2018-11-20 00:48:22 +00:00
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
2025-02-05 18:58:53 +00:00
2019-03-27 04:58:01 +00:00
2018-11-20 00:48:22 +00:00
# ---[ Python + Numpy
set ( onnxruntime_pybind_srcs_pattern
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / * . c c "
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / * . h "
)
2020-03-20 03:59:41 +00:00
if ( onnxruntime_ENABLE_TRAINING )
list ( APPEND onnxruntime_pybind_srcs_pattern
" $ { O R T T R A I N I N G _ R O O T } / o r t t r a i n i n g / p y t h o n / * . c c "
" $ { O R T T R A I N I N G _ R O O T } / o r t t r a i n i n g / p y t h o n / * . h "
)
endif ( )
2019-04-29 19:58:20 +00:00
file ( GLOB onnxruntime_pybind_srcs CONFIGURE_DEPENDS
$ { o n n x r u n t i m e _ p y b i n d _ s r c s _ p a t t e r n }
)
2021-09-02 21:26:58 +00:00
2021-08-27 23:23:35 +00:00
if ( onnxruntime_ENABLE_TRAINING )
list ( REMOVE_ITEM onnxruntime_pybind_srcs ${ ONNXRUNTIME_ROOT } /python/onnxruntime_pybind_module.cc )
endif ( )
2018-11-20 00:48:22 +00:00
2022-08-22 16:40:40 +00:00
# Add Pytorch as a library.
2023-04-10 23:00:04 +00:00
if ( onnxruntime_ENABLE_LAZY_TENSOR )
# Lazy Tensor requires Pytorch as a library.
2021-08-06 15:30:27 +00:00
list ( APPEND CMAKE_PREFIX_PATH ${ onnxruntime_PREBUILT_PYTORCH_PATH } )
2022-08-22 16:40:40 +00:00
# The following line may change ${CUDA_NVCC_FLAGS} and ${CMAKE_CUDA_FLAGS},
# if Pytorch is built from source.
# For example, pytorch/cmake/public/cuda.cmake and
# pytorch/torch/share/cmake/Caffe2/public/cuda.cmake both defines
# ONNX_NAMESPACE for both CUDA_NVCC_FLAGS and CMAKE_CUDA_FLAGS.
# Later, this ONNX_NAMESPACE may conflicts with ONNX_NAMESPACE set by ORT.
2021-08-06 15:30:27 +00:00
find_package ( Torch REQUIRED )
2022-08-22 16:40:40 +00:00
# Let's remove ONNX_NAMESPACE from Torch.
list ( FILTER CUDA_NVCC_FLAGS EXCLUDE REGEX "-DONNX_NAMESPACE=.+" )
string ( REGEX REPLACE "-DONNX_NAMESPACE=.+ " " " CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}" )
endif ( )
2021-08-06 15:30:27 +00:00
2022-08-22 16:40:40 +00:00
# Support ORT as a backend in Pytorch's LazyTensor.
if ( onnxruntime_ENABLE_LAZY_TENSOR )
file ( GLOB onnxruntime_lazy_tensor_extension_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ R O O T } / o r t t r a i n i n g / l a z y _ t e n s o r / * . c c " )
file ( GLOB onnxruntime_lazy_tensor_extension_headers CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ R O O T } / o r t t r a i n i n g / l a z y _ t e n s o r / * . h " )
if ( NOT MSVC )
set_source_files_properties ( ${ onnxruntime_lazy_tensor_extension_srcs } PROPERTIES COMPILE_FLAGS -Wno-unused-parameter )
set_source_files_properties ( ${ onnxruntime_lazy_tensor_extension_headers } PROPERTIES COMPILE_FLAGS -Wno-unused-parameter )
2021-08-06 15:30:27 +00:00
endif ( )
2021-09-02 21:26:58 +00:00
list ( APPEND onnxruntime_pybind_srcs
2022-08-22 16:40:40 +00:00
$ { o n n x r u n t i m e _ l a z y _ t e n s o r _ e x t e n s i o n _ s r c s } )
endif ( )
# onnxruntime_ENABLE_LAZY_TENSOR and onnxruntime_ENABLE_EAGER_MODE
# need DLPack code to pass tensors cross ORT and Pytorch boundary.
# TODO: consider making DLPack code a standalone library.
2023-04-10 23:00:04 +00:00
if ( onnxruntime_ENABLE_LAZY_TENSOR )
2022-08-22 16:40:40 +00:00
# If DLPack code is not built, add it to ORT's pybind target.
if ( NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP )
list ( APPEND onnxruntime_pybind_srcs
" $ { O R T T R A I N I N G _ R O O T } / o r t t r a i n i n g / c o r e / f r a m e w o r k / t o r c h / d l p a c k _ p y t h o n . c c " )
endif ( )
2021-08-06 15:30:27 +00:00
endif ( )
2021-02-04 16:38:56 +00:00
onnxruntime_add_shared_library_module ( onnxruntime_pybind11_state ${ onnxruntime_pybind_srcs } )
2022-08-22 16:40:40 +00:00
2020-02-04 03:33:14 +00:00
if ( MSVC )
2024-12-31 18:12:31 +00:00
# The following source file is only needed for the EPs that use delayloading. Namely, DML and WebGPU.
target_sources ( onnxruntime_pybind11_state PRIVATE "${ONNXRUNTIME_ROOT}/core/dll/delay_load_hook.cc" )
2020-02-04 03:33:14 +00:00
target_compile_options ( onnxruntime_pybind11_state PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /utf-8>" "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/utf-8>" )
2024-09-30 22:59:07 +00:00
target_compile_options ( onnxruntime_pybind11_state PRIVATE "/bigobj" )
2020-02-04 03:33:14 +00:00
endif ( )
2018-11-20 00:48:22 +00:00
if ( HAS_CAST_FUNCTION_TYPE )
2019-01-07 21:15:24 +00:00
target_compile_options ( onnxruntime_pybind11_state PRIVATE "-Wno-cast-function-type" )
2018-11-20 00:48:22 +00:00
endif ( )
2019-03-27 04:58:01 +00:00
2021-07-22 22:24:36 +00:00
# We export symbols using linker and the compiler does not know anything about it
# There is a problem with classes that have pybind types as members.
# See https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes
if ( NOT MSVC )
target_compile_options ( onnxruntime_pybind11_state PRIVATE "-fvisibility=hidden" )
endif ( )
2019-03-27 04:58:01 +00:00
if ( onnxruntime_PYBIND_EXPORT_OPSCHEMA )
target_compile_definitions ( onnxruntime_pybind11_state PRIVATE onnxruntime_PYBIND_EXPORT_OPSCHEMA )
2019-04-19 06:00:27 +00:00
endif ( )
2019-03-27 04:58:01 +00:00
2020-01-22 23:59:11 +00:00
if ( MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8 )
#TODO: fix the warnings
target_compile_options ( onnxruntime_pybind11_state PRIVATE "/wd4244" )
endif ( )
2021-03-26 23:26:42 +00:00
2025-02-05 18:58:53 +00:00
onnxruntime_add_include_to_target ( onnxruntime_pybind11_state Python::Module )
2021-06-03 06:36:49 +00:00
target_include_directories ( onnxruntime_pybind11_state PRIVATE ${ ONNXRUNTIME_ROOT } ${ pybind11_INCLUDE_DIRS } )
[CUDA] Fix cuda provider fallback inconsistency (#21425)
* Fix fallback setting (cuda still falls back to cuda).
* Fix cuda provider fallback inconsistent with/without CUDA_PATH
environment variable.
* Add cuda and cudnn major version requirement in error message.
Example result in Windows:
```
>>> import onnxruntime
>>> ort_session = onnxruntime.InferenceSession("model.onnx", providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
2024-07-19 17:43:44.2260019 [E:onnxruntime:Default, provider_bridge_ort.cc:1972 onnxruntime::TryGetProviderInfo_CUDA] D:\onnxruntime\onnxruntime\core\session\provider_bridge_ort.cc:1636 onnxruntime::ProviderLibrary::Get [ONNXRuntimeError] : 1 : FAIL : LoadLibrary failed with error 126 "" when trying to load "C:\Users\.conda\envs\py310\lib\site-packages\onnxruntime\capi\onnxruntime_providers_cuda.dll"
2024-07-19 17:43:44.2312351 [W:onnxruntime:Default, onnxruntime_pybind_state.cc:970 onnxruntime::python::CreateExecutionProviderInstance] Failed to create CUDAExecutionProvider. Require cuDNN 9.* and CUDA 12.*, and the latest MSVC runtime. Please install all dependencies as mentioned in the GPU requirements page (https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements), make sure they're in the PATH, and that your GPU is supported.
>>> ort_session
<onnxruntime.capi.onnxruntime_inference_collection.InferenceSession object at 0x0000016BB2DF7D60>
>>> ort_session.get_providers()
['CPUExecutionProvider']
```
Example result in Linux:
```
>>> import onnxruntime
>>> ort_session = onnxruntime.InferenceSession("resnet50-v2-7.onnx", providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
2024-07-20 20:33:26.486974543 [E:onnxruntime:Default, provider_bridge_ort.cc:1972 TryGetProviderInfo_CUDA] /work/onnxruntime/onnxruntime/core/session/provider_bridge_ort.cc:1636 onnxruntime::Provider& onnxruntime::ProviderLibrary::Get() [ONNXRuntimeError] : 1 : FAIL : Failed to load library libonnxruntime_providers_cuda.so with error: libcublasLt.so.12: cannot open shared object file: No such file or directory
2024-07-20 20:33:26.487034646 [W:onnxruntime:Default, onnxruntime_pybind_state.cc:961 CreateExecutionProviderInstance] Failed to create CUDAExecutionProvider. Require cuDNN 9.* and CUDA 12.*. Please install all dependencies as mentioned in the GPU requirements page (https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements), make sure they're in the PATH, and that your GPU is supported.
>>> ort_session.get_providers()
['CPUExecutionProvider']
```
### Motivation and Context
https://github.com/microsoft/onnxruntime/issues/21424
2024-07-23 18:58:04 +00:00
if ( onnxruntime_USE_CUDA )
Adding CUDNN Frontend and use for CUDA NN Convolution (#19470)
### Description
Added CUDNN Frontend and used it for NHWC convolutions, and optionally
fuse activation.
#### Backward compatible
- For model existed with FusedConv, model can still run.
- If ORT is built with cuDNN 8, cuDNN frontend will not be built into
binary. Old kernels (using cudnn backend APIs) are used.
#### Major Changes
- For cuDNN 9, we will enable cudnn frontend to fuse convolution and
bias when a provider option `fuse_conv_bias=1`.
- Remove the fusion of FusedConv from graph transformer for CUDA
provider, so there will not be FusedConv be added to graph for CUDA EP
in the future.
- Update cmake files regarding to cudnn settings. The search order of
CUDNN installation in build are like the following:
* environment variable `CUDNN_PATH`
* `onnxruntime_CUDNN_HOME` cmake extra defines. If a build starts from
build.py/build.sh, user can pass it through `--cudnn_home` parameter, or
by environment variable `CUDNN_HOME` if `--cudnn_home` not used.
* cudnn python package installation directory like
python3.xx/site-packages/nvidia/cudnn
* CUDA installation path
#### Potential Issues
- If ORT is built with cuDNN 8, FusedConv fusion is no longer done
automatically, so some model might have performance regression. If user
still wants FusedConv operator for performance reason, they can still
have multiple ways to walkaround: like use older version of onnxruntime;
or use older version of ORT to save optimized onnx, then run with latest
version of ORT. We believe that majority users have moved to cudnn 9
when 1.20 release (since the default in ORT and PyTorch is cudnn 9 for 3
months when 1.20 release), so the impact is small.
- cuDNN graph uses TF32 by default, and user cannot disable TF32 through
the use_tf32 cuda provider option. If user encounters accuracy issue
(like in testing), user has to set environment variable
`NVIDIA_TF32_OVERRIDE=0` to disable TF32. Need update the document of
use_tf32 later.
#### Follow ups
This is one of PRs that target to enable NHWC convolution in CUDA EP by
default if device supports it. There are other changes will follow up to
make it possible.
(1) Enable `prefer_nhwc` by default for device with sm >= 70.
(2) Change `fuse_conv_bias=1` by default after more testing.
(3) Add other NHWC operators (like Resize or UpSample).
### Motivation and Context
The new CUDNN Frontend library provides the functionality to fuse
operations and provides new heuristics for kernel selection. Here it
fuses the convolution with the pointwise bias operation. On the [NVIDIA
ResNet50](https://pytorch.org/hub/nvidia_deeplearningexamples_resnet50/)
we get a performance boost from 49.1144 ms to 42.4643 ms per inference
on a 2560x1440 input (`onnxruntime_perf_test -e cuda -I -q -r 100-d 1 -i
'prefer_nhwc|1' resnet50.onnx`).
---------
Co-authored-by: Tianlei Wu <tlwu@microsoft.com>
Co-authored-by: Maximilian Mueller <maximilianm@nvidia.com>
2024-08-02 22:16:42 +00:00
target_include_directories ( onnxruntime_pybind11_state PRIVATE ${ CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES } ${ CUDNN_INCLUDE_DIR } )
2020-07-21 14:28:13 +00:00
endif ( )
2022-09-22 21:53:40 +00:00
if ( onnxruntime_USE_CANN )
target_include_directories ( onnxruntime_pybind11_state PRIVATE ${ onnxruntime_CANN_HOME } /include )
endif ( )
2021-04-06 22:23:51 +00:00
if ( onnxruntime_USE_ROCM )
2022-11-03 11:32:30 +00:00
target_compile_options ( onnxruntime_pybind11_state PUBLIC -D__HIP_PLATFORM_AMD__=1 -D__HIP_PLATFORM_HCC__=1 )
2021-05-13 01:24:27 +00:00
target_include_directories ( onnxruntime_pybind11_state PRIVATE ${ onnxruntime_ROCM_HOME } /hipfft/include ${ onnxruntime_ROCM_HOME } /include ${ onnxruntime_ROCM_HOME } /hiprand/include ${ onnxruntime_ROCM_HOME } /rocrand/include ${ CMAKE_CURRENT_BINARY_DIR } /amdgpu/onnxruntime ${ CMAKE_CURRENT_BINARY_DIR } /amdgpu/orttraining )
endif ( )
2021-04-23 21:04:22 +00:00
if ( onnxruntime_USE_NCCL )
target_include_directories ( onnxruntime_pybind11_state PRIVATE ${ NCCL_INCLUDE_DIRS } )
endif ( )
2021-06-09 09:35:17 +00:00
2018-11-20 00:48:22 +00:00
if ( APPLE )
2024-11-05 00:30:50 +00:00
target_link_options ( onnxruntime_pybind11_state PRIVATE "LINKER:-exported_symbols_list,${ONNXRUNTIME_ROOT}/python/exported_symbols.lst" )
2018-11-20 00:48:22 +00:00
elseif ( UNIX )
2021-08-28 18:05:21 +00:00
if ( onnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS )
2024-11-05 00:30:50 +00:00
target_link_options ( onnxruntime_pybind11_state PRIVATE "LINKER:--version-script=${ONNXRUNTIME_ROOT}/python/version_script_expose_onnx_protobuf.lds" "LINKER:--gc-sections" )
2021-08-28 18:05:21 +00:00
else ( )
2024-08-30 19:17:26 +00:00
if ( NOT CMAKE_SYSTEM_NAME MATCHES "AIX" )
2024-11-05 00:30:50 +00:00
target_link_options ( onnxruntime_pybind11_state PRIVATE "LINKER:--version-script=${ONNXRUNTIME_ROOT}/python/version_script.lds" "LINKER:--gc-sections" )
2024-08-30 19:17:26 +00:00
endif ( )
2021-08-28 18:05:21 +00:00
endif ( )
2019-02-01 08:19:41 +00:00
else ( )
2024-11-05 00:30:50 +00:00
target_link_options ( onnxruntime_pybind11_state PRIVATE "-DEF:${ONNXRUNTIME_ROOT}/python/pybind.def" )
2018-11-20 00:48:22 +00:00
endif ( )
2022-06-09 08:07:30 +00:00
if ( onnxruntime_ENABLE_ATEN )
target_compile_definitions ( onnxruntime_pybind11_state PRIVATE ENABLE_ATEN )
2025-01-30 22:23:56 +00:00
endif ( )
if ( onnxruntime_ENABLE_DLPACK )
2025-02-06 18:10:31 +00:00
target_link_libraries ( onnxruntime_pybind11_state PRIVATE dlpack::dlpack )
2022-06-09 08:07:30 +00:00
endif ( )
2021-06-03 06:36:49 +00:00
if ( onnxruntime_ENABLE_TRAINING )
2021-06-19 05:41:07 +00:00
target_include_directories ( onnxruntime_pybind11_state PRIVATE ${ ORTTRAINING_ROOT } )
2021-06-03 06:36:49 +00:00
target_link_libraries ( onnxruntime_pybind11_state PRIVATE onnxruntime_training )
endif ( )
2022-08-22 16:40:40 +00:00
# Eager mode and LazyTensor are both Pytorch's backends, so their
# dependencies are set together below.
2023-04-10 23:00:04 +00:00
if ( onnxruntime_ENABLE_LAZY_TENSOR )
2022-08-22 16:40:40 +00:00
# Set library dependencies shared by aforementioned backends.
2021-08-06 15:30:27 +00:00
# todo: this is because the prebuild pytorch may use a different version of protobuf headers.
# force the build to find the protobuf headers ort using.
2022-08-22 16:40:40 +00:00
target_include_directories ( onnxruntime_pybind11_state PRIVATE
" $ { R E P O _ R O O T } / c m a k e / e x t e r n a l / p r o t o b u f / s r c "
$ { T O R C H _ I N C L U D E _ D I R S } )
2022-10-13 20:56:17 +00:00
# For eager mode, torch build has a mkl dependency from torch's cmake config,
# Linking to torch libraries to avoid this unnecessary mkl dependency.
target_include_directories ( onnxruntime_pybind11_state PRIVATE "${TORCH_INSTALL_PREFIX}/include" "${TORCH_INSTALL_PREFIX}/include/torch/csrc/api/include" )
find_library ( LIBTORCH_LIBRARY torch PATHS "${TORCH_INSTALL_PREFIX}/lib" )
find_library ( LIBTORCH_CPU_LIBRARY torch_cpu PATHS "${TORCH_INSTALL_PREFIX}/lib" )
find_library ( LIBC10_LIBRARY c10 PATHS "${TORCH_INSTALL_PREFIX}/lib" )
2022-08-22 16:40:40 +00:00
# Explicitly link torch_python to workaround https://github.com/pytorch/pytorch/issues/38122#issuecomment-694203281
find_library ( TORCH_PYTHON_LIBRARY torch_python PATHS "${TORCH_INSTALL_PREFIX}/lib" )
2022-10-13 20:56:17 +00:00
target_link_libraries ( onnxruntime_pybind11_state PRIVATE ${ LIBTORCH_LIBRARY } ${ LIBTORCH_CPU_LIBRARY } ${ LIBC10_LIBRARY } ${ TORCH_PYTHON_LIBRARY } )
if ( onnxruntime_USE_CUDA )
find_library ( LIBTORCH_CUDA_LIBRARY torch_cuda PATHS "${TORCH_INSTALL_PREFIX}/lib" )
find_library ( LIBC10_CUDA_LIBRARY c10_cuda PATHS "${TORCH_INSTALL_PREFIX}/lib" )
target_link_libraries ( onnxruntime_pybind11_state PRIVATE ${ LIBTORCH_CUDA_LIBRARY } ${ LIBC10_CUDA_LIBRARY } )
endif ( )
2022-08-22 16:40:40 +00:00
2022-06-09 08:07:30 +00:00
if ( MSVC )
2021-11-09 16:52:55 +00:00
target_compile_options ( onnxruntime_pybind11_state PRIVATE "/wd4100" "/wd4324" "/wd4458" "/wd4127" "/wd4193" "/wd4624" "/wd4702" )
2022-03-10 00:54:51 +00:00
target_compile_options ( onnxruntime_pybind11_state PRIVATE "/bigobj" "/wd4275" "/wd4244" "/wd4267" "/wd4067" )
2021-10-14 19:54:49 +00:00
endif ( )
2021-08-06 15:30:27 +00:00
endif ( )
2025-01-22 20:11:00 +00:00
set ( onnxruntime_pybind11_state_static_providers
2019-07-02 13:03:29 +00:00
$ { P R O V I D E R S _ N N A P I }
2024-12-02 21:57:30 +00:00
$ { P R O V I D E R S _ V S I N P U }
2022-08-11 02:12:51 +00:00
$ { P R O V I D E R S _ X N N P A C K }
2021-07-29 17:06:47 +00:00
$ { P R O V I D E R S _ C O R E M L }
Initial PR for RKNPU execution provider (#3609)
* Initial RKNPU execution provider
* Init
* Support Ops:
Conv, Relu, Clip, LeakyRelu,
MaxPool, AveragePool, GlobalAveragePool,
Concat, Softmax, BatchNormalization, Gemm,
Add, Mul, Sub,
Reshape, Squeeze, Unsqueeze,
Flatten, Transpose,
QLinearConv, DequantizeLinear
* Add rknpu unittest
* Update BUILD.md and Add RKNPU-ExecutionProvider.md
* misc code update
* fix CLIP accuracy issue.
* fix "Error: Duplicate definition of name".
* move rknpu_ddk out of onnxruntime submodule.
* remove temporary code.
* add rknpu namespace.
* update misc of node_attr_helper
* add const & comment for onnx_converter
* add const & comment for shaper
* unify variable name
Co-authored-by: dkm <dkm@rock-chips.com>
Co-authored-by: George Wu <jywu@microsoft.com>
2020-05-06 03:36:47 +00:00
$ { P R O V I D E R S _ R K N P U }
2019-10-15 13:13:07 +00:00
$ { P R O V I D E R S _ D M L }
2020-06-18 14:54:14 +00:00
$ { P R O V I D E R S _ A C L }
$ { P R O V I D E R S _ A R M N N }
2022-06-15 21:01:41 +00:00
$ { P R O V I D E R S _ X N N P A C K }
2024-10-08 23:10:46 +00:00
$ { P R O V I D E R S _ W E B G P U }
2023-01-11 20:25:04 +00:00
$ { P R O V I D E R S _ A Z U R E }
2025-01-22 20:11:00 +00:00
)
if ( onnxruntime_BUILD_QNN_EP_STATIC_LIB )
list ( APPEND onnxruntime_pybind11_state_static_providers PRIVATE onnxruntime_providers_qnn )
endif ( )
target_link_libraries ( onnxruntime_pybind11_state PRIVATE
o n n x r u n t i m e _ s e s s i o n
$ { o n n x r u n t i m e _ l i b s }
$ { o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e _ s t a t i c _ p r o v i d e r s }
2019-02-04 23:45:12 +00:00
o n n x r u n t i m e _ o p t i m i z e r
2018-11-20 00:48:22 +00:00
o n n x r u n t i m e _ p r o v i d e r s
2019-01-07 21:15:24 +00:00
o n n x r u n t i m e _ u t i l
2024-09-30 22:59:07 +00:00
o n n x r u n t i m e _ l o r a
2018-11-20 00:48:22 +00:00
o n n x r u n t i m e _ f r a m e w o r k
o n n x r u n t i m e _ u t i l
o n n x r u n t i m e _ g r a p h
2021-09-04 20:30:33 +00:00
$ { O N N X R U N T I M E _ M L A S _ L I B S }
2018-11-20 00:48:22 +00:00
o n n x r u n t i m e _ c o m m o n
2020-09-25 12:36:29 +00:00
o n n x r u n t i m e _ f l a t b u f f e r s
2020-02-04 03:33:14 +00:00
$ { p y b i n d 1 1 _ l i b }
2025-02-05 18:58:53 +00:00
P y t h o n : : N u m P y
2018-11-20 00:48:22 +00:00
)
set ( onnxruntime_pybind11_state_dependencies
$ { o n n x r u n t i m e _ E X T E R N A L _ D E P E N D E N C I E S }
2020-02-04 03:33:14 +00:00
$ { p y b i n d 1 1 _ d e p }
2018-11-20 00:48:22 +00:00
)
2024-11-05 00:30:50 +00:00
2018-11-20 00:48:22 +00:00
add_dependencies ( onnxruntime_pybind11_state ${ onnxruntime_pybind11_state_dependencies } )
2019-10-08 19:02:45 +00:00
2018-11-20 00:48:22 +00:00
if ( MSVC )
2024-11-05 00:30:50 +00:00
target_link_options ( onnxruntime_pybind11_state PRIVATE ${ onnxruntime_DELAYLOAD_FLAGS } )
2023-01-16 23:48:26 +00:00
# if MSVC, pybind11 undefines _DEBUG in pybind11/detail/common.h, which causes the pragma in pyconfig.h
# from the python installation to require the release version of the lib
# e.g. from a python 3.10 install:
# if defined(_DEBUG)
# pragma comment(lib,"python310_d.lib")
# elif defined(Py_LIMITED_API)
# pragma comment(lib,"python3.lib")
# else
# pragma comment(lib,"python310.lib")
# endif /* _DEBUG */
#
# See https://github.com/pybind/pybind11/issues/3403 for more background info.
#
# Explicitly use the release version of the python library to make the project file consistent with this.
target_link_libraries ( onnxruntime_pybind11_state PRIVATE ${ Python_LIBRARY_RELEASE } )
2018-11-29 02:29:16 +00:00
elseif ( APPLE )
2024-11-05 00:30:50 +00:00
# The following flag no longer works
#target_link_options(onnxruntime_pybind11_state PRIVATE "LINKER:-undefined,dynamic_lookup")
2018-11-29 04:01:21 +00:00
set_target_properties ( onnxruntime_pybind11_state PROPERTIES
I N S T A L L _ R P A T H " @ l o a d e r _ p a t h "
B U I L D _ W I T H _ I N S T A L L _ R P A T H T R U E
I N S T A L L _ R P A T H _ U S E _ L I N K _ P A T H F A L S E )
2018-11-20 00:48:22 +00:00
else ( )
2024-08-30 19:17:26 +00:00
if ( NOT CMAKE_SYSTEM_NAME MATCHES "AIX" )
2024-11-05 00:30:50 +00:00
target_link_options ( onnxruntime_pybind11_state PRIVATE "LINKER:-rpath=\$ORIGIN" )
2024-08-30 19:17:26 +00:00
endif ( )
2018-11-20 00:48:22 +00:00
endif ( )
2021-08-28 18:05:21 +00:00
if ( onnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS )
set ( onnxruntime_CUSTOM_EXTERNAL_LIBRARIES "${onnxruntime_EXTERNAL_LIBRARIES}" )
list ( FIND onnxruntime_CUSTOM_EXTERNAL_LIBRARIES onnx ONNX_INDEX )
list ( FIND onnxruntime_CUSTOM_EXTERNAL_LIBRARIES ${ PROTOBUF_LIB } PROTOBUF_INDEX )
MATH ( EXPR PROTOBUF_INDEX_NEXT "${PROTOBUF_INDEX} + 1" )
if ( ONNX_INDEX GREATER_EQUAL 0 AND PROTOBUF_INDEX GREATER_EQUAL 0 )
# Expect protobuf to follow onnx due to dependence
2024-11-05 00:30:50 +00:00
list ( INSERT onnxruntime_CUSTOM_EXTERNAL_LIBRARIES ${ ONNX_INDEX } "LINKER:--no-as-needed" )
list ( INSERT onnxruntime_CUSTOM_EXTERNAL_LIBRARIES ${ PROTOBUF_INDEX_NEXT } "LINKER:--as-needed" )
2021-08-28 18:05:21 +00:00
else ( )
message ( FATAL_ERROR "Required external libraries onnx and protobuf are not found in onnxruntime_EXTERNAL_LIBRARIES" )
endif ( )
target_link_libraries ( onnxruntime_pybind11_state PRIVATE ${ onnxruntime_CUSTOM_EXTERNAL_LIBRARIES } )
else ( )
target_link_libraries ( onnxruntime_pybind11_state PRIVATE ${ onnxruntime_EXTERNAL_LIBRARIES } )
endif ( )
2021-06-03 06:36:49 +00:00
2018-11-20 00:48:22 +00:00
set_target_properties ( onnxruntime_pybind11_state PROPERTIES PREFIX "" )
set_target_properties ( onnxruntime_pybind11_state PROPERTIES FOLDER "ONNXRuntime" )
2019-05-16 21:06:38 +00:00
if ( onnxruntime_ENABLE_LTO )
set_target_properties ( onnxruntime_pybind11_state PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE )
set_target_properties ( onnxruntime_pybind11_state PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE )
2020-09-10 20:50:28 +00:00
set_target_properties ( onnxruntime_pybind11_state PROPERTIES INTERPROCEDURAL_OPTIMIZATION_MINSIZEREL TRUE )
2019-05-16 21:06:38 +00:00
endif ( )
2018-11-20 00:48:22 +00:00
if ( MSVC )
set_target_properties ( onnxruntime_pybind11_state PROPERTIES SUFFIX ".pyd" )
else ( )
set_target_properties ( onnxruntime_pybind11_state PROPERTIES SUFFIX ".so" )
endif ( )
2021-02-21 23:11:28 +00:00
# Generate version_info.py in Windows build.
# Has to be done before onnxruntime_python_srcs is set.
if ( WIN32 )
set ( VERSION_INFO_FILE "${ONNXRUNTIME_ROOT}/python/version_info.py" )
if ( onnxruntime_USE_CUDA )
file ( WRITE "${VERSION_INFO_FILE}" "use_cuda = True\n" )
2022-08-25 01:21:50 +00:00
if ( onnxruntime_CUDNN_HOME )
file ( GLOB CUDNN_DLL_PATH "${onnxruntime_CUDNN_HOME}/bin/cudnn64_*.dll" )
if ( NOT CUDNN_DLL_PATH )
message ( FATAL_ERROR "cuDNN not found in ${onnxruntime_CUDNN_HOME}" )
endif ( )
else ( )
file ( GLOB CUDNN_DLL_PATH "${onnxruntime_CUDA_HOME}/bin/cudnn64_*.dll" )
if ( NOT CUDNN_DLL_PATH )
message ( FATAL_ERROR "cuDNN not found in ${onnxruntime_CUDA_HOME}" )
endif ( )
2021-02-21 23:11:28 +00:00
endif ( )
get_filename_component ( CUDNN_DLL_NAME ${ CUDNN_DLL_PATH } NAME_WE )
string ( REPLACE "cudnn64_" "" CUDNN_VERSION "${CUDNN_DLL_NAME}" )
2023-05-02 01:00:47 +00:00
if ( NOT onnxruntime_CUDA_VERSION )
2024-02-27 19:26:48 +00:00
set ( onnxruntime_CUDA_VERSION ${ CUDAToolkit_VERSION } )
2023-05-02 01:00:47 +00:00
message ( "onnxruntime_CUDA_VERSION=${onnxruntime_CUDA_VERSION}" )
endif ( )
2021-02-21 23:11:28 +00:00
file ( APPEND "${VERSION_INFO_FILE}"
" c u d a _ v e r s i o n = \ " $ { o n n x r u n t i m e _ C U D A _ V E R S I O N } \ " \ n "
" c u d n n _ v e r s i o n = \ " $ { C U D N N _ V E R S I O N } \ " \ n "
)
else ( )
file ( WRITE "${VERSION_INFO_FILE}" "use_cuda = False\n" )
endif ( )
if ( "${MSVC_TOOLSET_VERSION}" STREQUAL "142" )
file ( APPEND "${VERSION_INFO_FILE}" "vs2019 = True\n" )
else ( )
file ( APPEND "${VERSION_INFO_FILE}" "vs2019 = False\n" )
endif ( )
endif ( )
2019-04-29 19:58:20 +00:00
file ( GLOB onnxruntime_backend_srcs CONFIGURE_DEPENDS
2018-11-20 00:48:22 +00:00
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / b a c k e n d / * . p y "
)
2020-03-20 03:59:41 +00:00
if ( onnxruntime_ENABLE_TRAINING )
file ( GLOB onnxruntime_python_srcs CONFIGURE_DEPENDS
2018-11-20 00:48:22 +00:00
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / * . p y "
2020-03-20 03:59:41 +00:00
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / * . p y "
)
else ( )
file ( GLOB onnxruntime_python_srcs CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / * . p y "
)
endif ( )
2021-09-02 16:54:32 +00:00
# Generate _pybind_state.py from _pybind_state.py.in replacing macros with either setdlopenflags or ""
if ( onnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS )
set ( ONNXRUNTIME_SETDLOPENFLAGS_GLOBAL "sys.setdlopenflags(os.RTLD_GLOBAL|os.RTLD_NOW|os.RTLD_DEEPBIND)" )
set ( ONNXRUNTIME_SETDLOPENFLAGS_LOCAL "sys.setdlopenflags(os.RTLD_LOCAL|os.RTLD_NOW|os.RTLD_DEEPBIND)" )
else ( )
set ( ONNXRUNTIME_SETDLOPENFLAGS_GLOBAL "" )
set ( ONNXRUNTIME_SETDLOPENFLAGS_LOCAL "" )
endif ( )
2022-08-22 16:40:40 +00:00
if ( onnxruntime_ENABLE_LAZY_TENSOR )
# Import torch so that onnxruntime's pybind can see its DLLs.
set ( ONNXRUNTIME_IMPORT_PYTORCH_TO_RESOLVE_DLLS "import torch" )
else ( )
set ( ONNXRUNTIME_IMPORT_PYTORCH_TO_RESOLVE_DLLS "" )
endif ( )
2021-09-02 16:54:32 +00:00
configure_file ( ${ ONNXRUNTIME_ROOT } /python/_pybind_state.py.in
$ { C M A K E _ B I N A R Y _ D I R } / o n n x r u n t i m e / c a p i / _ p y b i n d _ s t a t e . p y )
2020-03-20 03:59:41 +00:00
if ( onnxruntime_ENABLE_TRAINING )
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
file ( GLOB onnxruntime_python_root_srcs CONFIGURE_DEPENDS
2020-09-09 16:46:06 +00:00
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / * . p y "
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
)
file ( GLOB onnxruntime_python_amp_srcs CONFIGURE_DEPENDS
2020-09-09 16:46:06 +00:00
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / a m p / * . p y "
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
)
2022-02-18 22:00:49 +00:00
file ( GLOB onnxruntime_python_experimental_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / e x p e r i m e n t a l / * . p y "
)
file ( GLOB onnxruntime_python_gradient_graph_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / e x p e r i m e n t a l / g r a d i e n t _ g r a p h / * . p y "
)
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
file ( GLOB onnxruntime_python_optim_srcs CONFIGURE_DEPENDS
2020-09-09 16:46:06 +00:00
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o p t i m / * . p y "
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
)
2021-04-26 21:53:50 +00:00
file ( GLOB onnxruntime_python_ortmodule_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / * . p y "
)
2021-07-30 20:05:32 +00:00
file ( GLOB onnxruntime_python_ortmodule_experimental_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / * . p y "
)
file ( GLOB onnxruntime_python_ortmodule_experimental_json_config_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / j s o n _ c o n f i g / * . p y "
)
2021-09-28 00:18:22 +00:00
file ( GLOB onnxruntime_python_ortmodule_experimental_hierarchical_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / h i e r a r c h i c a l _ o r t m o d u l e / * . p y "
)
2021-06-29 01:11:58 +00:00
file ( GLOB onnxruntime_python_ortmodule_torch_cpp_ext_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / * . p y "
)
file ( GLOB onnxruntime_python_ortmodule_torch_cpp_ext_aten_op_executor_srcs CONFIGURE_DEPENDS
2022-06-09 08:07:30 +00:00
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o r c h _ c p p _ e x t e n s i o n s / a t e n _ o p _ e x e c u t o r / * "
2021-06-29 01:11:58 +00:00
)
2021-09-01 01:29:26 +00:00
file ( GLOB onnxruntime_python_ortmodule_torch_cpp_ext_torch_interop_utils_srcs CONFIGURE_DEPENDS
2021-09-30 14:37:35 +00:00
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c p u / t o r c h _ i n t e r o p _ u t i l s / * "
2021-09-01 01:29:26 +00:00
)
2021-06-29 01:11:58 +00:00
file ( GLOB onnxruntime_python_ortmodule_torch_cpp_ext_torch_gpu_allocator_srcs CONFIGURE_DEPENDS
2021-09-30 14:37:35 +00:00
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c u d a / t o r c h _ g p u _ a l l o c a t o r / * "
2021-06-29 01:11:58 +00:00
)
2021-10-26 05:13:49 +00:00
file ( GLOB onnxruntime_python_ortmodule_torch_cpp_ext_fused_ops_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c u d a / f u s e d _ o p s / * "
2021-10-06 03:50:34 +00:00
)
2023-10-27 02:29:27 +00:00
file ( GLOB onnxruntime_python_ortmodule_graph_optimizers_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / g r a p h _ o p t i m i z e r s / * "
)
2024-04-18 18:30:15 +00:00
file ( GLOB onnxruntime_python_ortmodule_pipe_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / p i p e / * "
)
2023-07-13 10:17:58 +00:00
file ( GLOB onnxruntime_python_ort_triton_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t _ t r i t o n / * . p y "
)
file ( GLOB onnxruntime_python_ort_triton_kernel_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o r t _ t r i t o n / k e r n e l / * . p y "
)
2023-08-04 05:58:21 +00:00
file ( GLOB onnxruntime_python_utils_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / u t i l s / * . p y "
)
2022-02-14 21:46:14 +00:00
file ( GLOB onnxruntime_python_utils_data_srcs CONFIGURE_DEPENDS
2023-07-13 10:17:58 +00:00
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / u t i l s / d a t a / * "
2022-02-14 21:46:14 +00:00
)
Statistics tool for ORTModule convergence parity (#15020)
### Statistics tool for ORTModule convergence parity
As ORTModule get more and more validated, it is pretty fast to
intergrade PyTorch based model with ORT.
The same time, we need make sure once there is convergence issue, we
don't spend months of time to investigate. As part of this efforts, this
PR is introducing a tool to dump activation statistics without much
involvement from users. The dumping results contains only some statistic
numbers plus sampled data, which is not big, compared with dumping all
the tensors, it is much faster and space efficient.
For us to use it, two single lines are needed before wrapping ORTModule.
For baseline run, need also apply the same trick.
```
+ from onnxruntime.training.utils.hooks import SubscriberManager, StatisticsSubscriber
+ SubscriberManager.subscribe(model, [StatisticsSubscriber("pt_out", override_output_dir=True)])
```
Once you run the steps, following command can be used to merge result
into per-step-summary respectively for ORT and baseline runs.
```bash
python -m onnxruntime.training.utils.hooks.merge_activation_summary --pt_dir pt_out --ort_dir ort_out --output_dir /tmp/output
```
Docs is added here as part of this PR [convergence investigation
notes](https://github.com/microsoft/onnxruntime/blob/pengwa/conv_tool/docs/ORTModule_Convergence_Notes.md)
Based on the generated merged files, we can compare them with tools.
### Design and Implementation
This PR introduced a common mechanism registering custom logic for
nn.Module's post forward hooks. And statistics for activation
(StatisticsSubscriber) is one of the implementations. If there is other
needs, we can define another XXSubscriber to do the customized things.
2023-03-23 12:34:24 +00:00
file ( GLOB onnxruntime_python_utils_hooks_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / u t i l s / h o o k s / * "
)
2023-01-03 21:28:16 +00:00
if ( onnxruntime_ENABLE_TRAINING_APIS )
2022-05-25 01:21:39 +00:00
file ( GLOB onnxruntime_python_onnxblock_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o n n x b l o c k / * "
)
file ( GLOB onnxruntime_python_onnxblock_loss_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o n n x b l o c k / l o s s / * "
)
2022-09-16 16:38:24 +00:00
file ( GLOB onnxruntime_python_api_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / a p i / * "
)
2022-05-25 01:21:39 +00:00
file ( GLOB onnxruntime_python_onnxblock_optim_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / p y t h o n / t r a i n i n g / o n n x b l o c k / o p t i m / * "
)
endif ( )
2020-03-20 03:59:41 +00:00
endif ( )
2021-02-24 04:21:57 +00:00
if ( onnxruntime_BUILD_UNIT_TESTS )
file ( GLOB onnxruntime_python_test_srcs CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / t e s t / p y t h o n / * . p y "
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / t e s t / p y t h o n / * . p y "
2021-02-27 04:25:23 +00:00
" $ { O R T T R A I N I N G _ S O U R C E _ D I R } / t e s t / p y t h o n / * . j s o n "
2021-02-24 04:21:57 +00:00
)
file ( GLOB onnxruntime_python_quantization_test_srcs CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / t e s t / p y t h o n / q u a n t i z a t i o n / * . p y "
)
2021-06-09 02:43:59 +00:00
file ( GLOB onnxruntime_python_transformers_test_srcs CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / t e s t / p y t h o n / t r a n s f o r m e r s / * . p y "
)
file ( GLOB onnxruntime_python_transformers_testdata_srcs CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / t e s t / p y t h o n / t r a n s f o r m e r s / t e s t _ d a t a / m o d e l s / * . o n n x "
)
Whisper Model Optimization (#15473)
### Description
This PR contains fusion-level and kernel-level optimizations for
[OpenAI's Whisper](https://github.com/openai/whisper).
Some of the added optimizations include:
- Pruning of duplicate/unnecessary inputs and outputs
- Fusion support for Whisper models with or without these inputs/outputs
(e.g. with these inputs/outputs if exporting with an older official
Optimum version, without these inputs/outputs if exporting with Optimum
from source)
- Attention fusions
- For Whisper's encoder and decoder
- Modified symbolic shape inference for present output when no past
input exists (for decoder)
- Multi-head attention fusions
- For Whisper's decoder and decoder with past
- Packed MatMul for the 3 MatMuls excluded in multi-head attention
fusion
- Attention kernel changes
- CPU:
- Different Q and KV sequence lengths
- Parallel memset for large sequence lengths
- Convert broadcast add after MatMul of Q and K (add_qk) to element-wise
add
- Separate present key-value output into present key and present value
(for multi-head attention spec)
- CUDA:
- Use memory efficient attention compute kernel with present state (for
decoder)
- Multi-head attention kernel changes
- CPU:
- Introduction of multi-head attention CPU kernel (previously did not
exist)
- Use AddBiasReshape instead of AddBiasTranspose when sequence length =
1 (for decoder with past)
- Different Q, K, V input shapes
- Pass past key and past value directly as key and value
- CUDA:
- Use memory efficient attention compute kernel with past and/or present
state (for decoder with past)
### Usage
To use the optimizations, run the ORT transformer optimizer script as
follows:
```
$ cd onnxruntime/onnxruntime/python/tools/transformers/
$ python3 optimizer.py --input <filename>.onnx --output <filename>.onnx --model_type bart --num_heads <number of attention heads, depends on the size of the whisper model used> --hidden_size <attention hidden size, depends on the size of the whisper model used> --use_external_data_format --use_multi_head_attention
```
Once optimized, here's an example of how to run Whisper with [Hugging
Face's Optimum](https://github.com/huggingface/optimum):
```
from transformers.onnx.utils import get_preprocessor
from optimum.onnxruntime import ORTModelForSpeechSeq2Seq
from optimum.pipelines import pipeline as ort_pipeline
import whisper # Installed from OpenAI's repo - setup instructions at https://github.com/openai/whisper/
directory = './whisper_opt' # Where the optimized ONNX models are located
model_name = 'openai/whisper-tiny'
device = 'cpu'
# Get pipeline
processor = get_preprocessor(model_name)
model = ORTModelForSpeechSeq2Seq.from_pretrained(
directory,
use_io_binding=(device == 'cuda'),
provider='CPUExecutionProvider',
).to(device)
pipe = ort_pipeline(
"automatic-speech-recognition",
model=model,
tokenizer=processor.tokenizer,
feature_extractor=processor.feature_extractor,
device=(-1 if device == 'cpu' else 0),
)
# Load audio file and run pipeline
audio = whisper.load_audio('tests/jfk.flac')
audio = whisper.pad_or_trim(audio)
outputs = pipe([audio])
print(outputs)
```
Note: In order to use these changes with Optimum, it is recommended to
use Optimum from source to have the following changes:
- https://github.com/huggingface/optimum/pull/872
- https://github.com/huggingface/optimum/pull/920
### Motivation and Context
This PR helps the following issues:
- https://github.com/microsoft/onnxruntime/issues/15100
- https://github.com/microsoft/onnxruntime/issues/15235
- https://github.com/huggingface/optimum/issues/869 (work in progress)
This PR can be used with the other currently merged Whisper PRs:
- https://github.com/microsoft/onnxruntime/pull/15247
- https://github.com/microsoft/onnxruntime/pull/15339
- https://github.com/microsoft/onnxruntime/pull/15362
- https://github.com/microsoft/onnxruntime/pull/15365
- https://github.com/microsoft/onnxruntime/pull/15427
This PR uses changes from the following merged PRs:
- https://github.com/microsoft/onnxruntime/pull/14198
- https://github.com/microsoft/onnxruntime/pull/14146
- https://github.com/microsoft/onnxruntime/pull/14201
- https://github.com/microsoft/onnxruntime/pull/14928 (this introduced
the new multi-head attention spec)
2023-04-19 00:13:54 +00:00
file ( GLOB onnxruntime_python_transformers_testdata_whisper CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / t e s t / p y t h o n / t r a n s f o r m e r s / t e s t _ d a t a / m o d e l s / w h i s p e r / * . o n n x "
)
2023-11-19 07:39:04 +00:00
file ( GLOB onnxruntime_python_transformers_testdata_conformer CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / t e s t / p y t h o n / t r a n s f o r m e r s / t e s t _ d a t a / m o d e l s / c o n f o r m e r / * . o n n x "
)
2021-02-24 04:21:57 +00:00
endif ( )
2019-04-29 19:58:20 +00:00
file ( GLOB onnxruntime_python_tools_srcs CONFIGURE_DEPENDS
2018-11-20 00:48:22 +00:00
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / * . p y "
)
2020-07-09 04:42:53 +00:00
file ( GLOB onnxruntime_python_quantization_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / q u a n t i z a t i o n / * . p y "
)
2020-09-01 16:07:46 +00:00
file ( GLOB onnxruntime_python_quantization_operators_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / q u a n t i z a t i o n / o p e r a t o r s / * . p y "
)
2021-03-19 08:09:11 +00:00
file ( GLOB onnxruntime_python_quantization_cal_table_flatbuffers_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / q u a n t i z a t i o n / C a l T a b l e F l a t B u f f e r s / * . p y "
)
2023-12-12 16:43:04 +00:00
file ( GLOB onnxruntime_python_quantization_fusions_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / q u a n t i z a t i o n / f u s i o n s / * . p y "
)
[Quantization] Tensor quant overrides and QNN EP quantization configuration (#18465)
### Description
#### 1. Adds `TensorQuantOverrides` extra option
Allows specifying a dictionary of tensor-level quantization overrides:
```
TensorQuantOverrides = dictionary :
Default is {}. Set tensor quantization overrides. The key is a tensor name and the value is a
list of dictionaries. For per-tensor quantization, the list contains a single dictionary. For
per-channel quantization, the list contains a dictionary for each channel in the tensor.
Each dictionary contains optional overrides with the following keys and values.
'quant_type' = QuantType : The tensor's quantization data type.
'scale' = Float : The scale value to use. Must also specify `zero_point` if set.
'zero_point' = Int : The zero-point value to use. Must also specify `scale` is set.
'symmetric' = Bool : If the tensor should use symmetric quantization. Invalid if also
set `scale` or `zero_point`.
'reduce_range' = Bool : If the quantization range should be reduced. Invalid if also
set `scale` or `zero_point`.
'rmax' = Float : Override the maximum real tensor value in calibration data.
Invalid if also set `scale` or `zero_point`.
'rmin' = Float : Override the minimum real tensor value in calibration data.
Invalid if also set `scale` or `zero_point`.
```
- All of the options are optional.
- Some combinations are invalid.
- Ex: `rmax` and `rmin` are unnecessary if the `zero_point` and `scale`
are also specified.
Example for per-tensor quantization overrides:
```Python3
extra_options = {
"TensorQuantOverrides": {
"SIG_OUT": [{"scale": 1.0, "zero_point": 127}],
"WGT": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
"BIAS": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
},
}
```
Example for per-channel quantization overrides (Conv weight and bias):
```Python3
extra_options = {
"TensorQuantOverrides": {
"WGT": [
{
"quant_type": quantization.QuantType.QUInt8,
"rmin": 0.0,
"rmax": 2.5,
"reduce_range": True,
},
{
"quant_type": quantization.QuantType.QUInt8,
"rmin": 0.2,
"rmax": 2.55,
"reduce_range": False,
},
],
"BIAS": [
{"zero_point": 0, "scale": 0.000621},
{"zero_point": 0, "scale": 0.23},
],
},
}
```
#### 2. Adds utilities to get the default QDQ configs for QNN EP
Added a `quantization.execution_providers.qnn.get_qnn_qdq_config` method
that inspects the model and returns suitable quantization
configurations.
Example usage:
```python3
from quantization import quantize, QuantType
from quantization.execution_providers.qnn import get_qnn_qdq_config
qnn_config = get_qnn_qdq_config(input_model_path,
data_reader,
activation_type=QuantType.QUInt16,
weight_type=QuantType.QUInt8)
quantize(input_model_path,
output_model_path,
qnn_config)
```
### Motivation and Context
Make it possible to create more QDQ models that run on QNN EP.
---------
Signed-off-by: adrianlizarraga <adlizarraga@microsoft.com>
2023-12-05 01:54:58 +00:00
file ( GLOB onnxruntime_python_quantization_ep_qnn_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / q u a n t i z a t i o n / e x e c u t i o n _ p r o v i d e r s / q n n / * . p y "
)
2020-09-10 22:42:15 +00:00
file ( GLOB onnxruntime_python_transformers_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / * . p y "
)
2023-02-07 15:49:15 +00:00
file ( GLOB onnxruntime_python_transformers_models_bart_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / b a r t / * . p y "
)
file ( GLOB onnxruntime_python_transformers_models_bert_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / b e r t / * . p y "
)
2022-04-20 18:09:26 +00:00
file ( GLOB onnxruntime_python_transformers_models_gpt2_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / g p t 2 / * . p y "
)
2023-08-23 01:05:11 +00:00
file ( GLOB onnxruntime_python_transformers_models_llama_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / l l a m a / * . p y "
)
2022-04-10 05:35:14 +00:00
file ( GLOB onnxruntime_python_transformers_models_longformer_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / l o n g f o r m e r / * . p y "
)
2024-02-05 18:15:16 +00:00
file ( GLOB onnxruntime_python_transformers_models_phi2_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / p h i 2 / * . p y "
)
2024-09-18 21:31:59 +00:00
file ( GLOB onnxruntime_python_transformers_models_sam2_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / s a m 2 / * . p y "
)
2023-02-07 15:49:15 +00:00
file ( GLOB onnxruntime_python_transformers_models_stable_diffusion_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / s t a b l e _ d i f f u s i o n / * . p y "
)
2022-04-10 05:35:14 +00:00
file ( GLOB onnxruntime_python_transformers_models_t5_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / t 5 / * . p y "
Sync ORTModule branch with master and fix tests (#6526)
* Deprecate Python global configuration functions [Part 1] (#5923)
Enable options to be set via execution provider (EP)-specific options and log deprecation warning from current global configuration functions.
* remove dnnl_dll_path from post build copy (#6142)
* Model Fusion For Bart (#6105)
Fusion fix for Bart models
* Unify IExecutionProvider and IExecutionProviderFactory interfaces (#6108)
* Remove Provider_IExecutionProvider and make the internal IExecutionProvider usable by shared providers
* Change Provider_IExecutionProviderFactory to be the core version.
* Enable running the mnist_training sample without cuda (#6085)
Signed-off-by: George Nash <george.nash@intel.com>
* nnapi add min max support (#6117)
* Fix CUDA test hang: (#6138)
- Make condition check in `CUDAAllocatorTest` to ensure CUDA device is present.
* Fix TensorRT kernel conflict issue for subgraphs of control flow operators (#6115)
* add static subgraph kernel index
* change kernel naming to avoid conflicts
* Add gradient registration for Abs. (#6139)
* Partition initial optimizer state for Zero-1 (#6093)
* Initial changes
* Working changes
* Working changes
* Cleanup
* fix windows CI
* Review comments
* review comments
* Fix edge case in BFCArena where allocation failures could lead to an infinite loop. (#6145)
#4656
* Revert "work around of the build break in mac (#6069)" (#6150)
This reverts commit 3cae28699bed5de1fcaadb219fa69bae0fc3cee8.
* Fix clean_docker_image_cache.py detection of image pushes. (#6151)
Fix clean_docker_image_cache.py detection of image pushes. They were being ignored because the expected HTTP status code was wrong. For pushes, it's 201 instead of 200.
* MLAS: add NEON version of int8 depthwise convolution (#6152)
* Using a map of of ops to stages as input of partition function. (#5940)
* New partition algorithm running before AD
* Convert cut_group_info into device map. Work in progress -- works for bert-tiny with pp=2
* Removing code for partition of bwd graphs
* Remove old code
* Adding some verification code
* Handle Shared Initializer
* Renaming rank with stage
* Added first unit test
* new test
* redundant check
* undo change in bert
* Moved cut-based partition to testing utils file
Co-authored-by: xzhu1900
Co-authored-by: wschin
* New conversion function and tests
* minor
* remove test that is not needed2
* improve GetDeviceAssignment and PR comments
* minor changes
* PR comments
* improving documentation and variable naming
* add documentation
* Variable naming and docs
* more doc improvements
* more doc improvements
* missing static cast
* Fix test file for windows
* Fix test file for windows
* Fix test file for windows
* stage id is not the same as rank id
* PR comments
* PR comments
* More comments
* More comments
* Minor fix to satisfy c++14 (#6162)
* Deprecating Horovod and refactored Adasum computations (#5468)
deprecated horovod submodule
refactored adasum logic to be ort-native
added tests for native kernel and e2e tests
* Update TensorRT-ExecutionProvider.md (#6161)
* Bugfix for topk cuda kernel (#6164)
* fix the issue that std::numeric_limits cannot handle half type
* adding a test
Co-authored-by: Du Li <duli@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Revert "Fuse MatMulIntegerToFloat only when scales are scalar (#6008)" (#6169)
This reverts commit f2dcba7afe0d42ebdaaef0c6cdf913a1156c9e98.
* Remove ignored build warnings for pybind on Mac (#6165)
* save_checkpoint, load_checkpoint and aggregate_checkpoints (#6136)
* save_checkpoint and load_checkpoint implementations
* checkpoint aggregation logic
* unit tests for save_checkpoint, load_checkpoint and aggregate_checkpoints
* Don't try to bind unused inputs in the Training frontend (#6166)
* Update documentation for contributing a PR and add deprecation notices for PyOp and ORT server. (#6172)
* aggregate model states only for the case when mixed precision was true (#6176)
* [NNAPI EP] Enable per-channel quantization for QlinearConv (#6155)
* Enable qlinearconv per-channel quantization
* Fix the android CI test failure
* Add Android Version Check for Per-Channel Quant
* Address PR comments
* Fix some minor issues
* Add verification of per-channel zero points
* Make the error tolerance configurable
* Fix typo in BERT pretraining script (#6175)
A misplaced `}` meant that the `'enable_adasum'` option was interpreted incorrectly, causing the test to fail.
* Update get_docker_image.py to enable use without image cache container registry. (#6177)
Update get_docker_image.py to enable use without image cache container registry.
* Helper for compiling EP to generate deterministic unique ids for use in MetaDef names (#6156)
* Create a helper for generating unique ids that can be used by an EP that creates compiled nodes and needs ids to be deterministic for a model when used in multiple sessions.
Added to IExecutionProvider as this can potentially be used by all compiling EPs and is more robust than a simplistic counter (although EP implementer is free to choose either approach).
* Restructure the helper so it can be called across the EP bridge.
Add ability to call id generation helper from EP bridge
- convert DNNL EP to use helper to validate
Address issue where a new Model may be loaded into the same address as a previous one.
- hash the bytes in the Graph instance (1728 bytes currently) to use as the key to the full hash for the model
Add lock around id generation to ensure no issues if multiple sessions partitions graphs at exactly the same time.
- Extremely unlikely but would be hard to debug and the locking cost is not an issue as it's only incurred during graph partitioning and not execution.
* Backend APIs for checkpointing (#5803)
* Add backend API GetOptimizerState and GetModelState
* add GetPartitionInfoMap
* Android coverage dashboard (#6163)
* Write the report to a file.
* Post code coverage to the Dashboard database.
* Add usage details of unified MCR container image (#6182)
Going forward, a single unifed docker image will be published in
MCR. The hardware accelerator target choice will have to be made
in the application using OpenVINO EP's runtime config options.
* improve perf for softmax (#6128)
* improve perf for both gathergrad and softmax
* revert the change in gathergrad and will be done in another PR.
* address comments from code review.
* Tune fast Gelu to use exp(x) instead of tanh(x) on Rocm platform (#6174)
* tune fast gelu to use exp(x) instead of tanh(x) on rocm
* update to use expression 2/(1+exp(-2x))-1 for stability
* Add Status.csv to EP Perf Tool (#6167)
* merge master, keep postprocess status commit
* download float16.py everytime
* removing hardcoded values
* Lochi/quantization tool for trt (#6103)
* Initial implementation of generating calibration dynamic range table
* Initialize validation support for Quantization
* Initialize validation support for Quantization (cont.)
* Improve validation support for Quantization
* Improve validation support for Quantization
* Rewrite/Refine for calibration and validation
* Rewrite/Refine for calibration and validation (cont.)
* Refine code
* Refine code
* Add data reader for BERT
* Add flatbuffers to serialize calibration table
* Refine code and add BERT evaluation
* Refine the code
* minor modification
* Add preprocess/postprocess of vision team yolov3 and refine the code
* Update annotation
* Make bbox cooridates more accurate
* Fix bug
* Add support of batch processing
* Batch processing for model zoo yolov3
* Add batch inference for evaluation
* Refine the code
* Add README
* Add comments
* Refine the code for PR
* Remove batch support checking in data_reader and refine the code
* Refine the code for PR
* Refine the code for PR review
Co-authored-by: Olivia Jain <oljain@microsoft.com>
* Implement ScatterND for CUDA EP (#6184)
* Condition fix in Resize operator (#6193)
* Clean up checkpoint tests to use the new checkpoint functions (#6188)
* add deprecation warning for old checkpoint functions
* update all the distributed checkpoint tests to use new checkpoint functions
* Implement comparing outputs that are sequence of maps of strings to floats (#6180)
* Implement conversion from ortvalue to Itensor for string tensors and comparing sequence of maps of strings to floats
* PR comments
* Dockerfile to build onnxruntime with ROCm 4.0
* Add ability to skip GPU tests based on GPU adapter name (#6198)
* Implement conversion from ortvalue to Itensor for string tensors and comparing sequence of maps of strings to floats
* PR comments
* Add ability to skip gpu tests according to adapter description
* spacing
* spacing
* spacing
* Openvino ep 2021.2 (#6196)
* Enabling fasterrcnn variant and vehicle detector
* changes for 2021_2 branch
* yolov3_pytorch commit
* fixed braces in basic_backend.cc
* ci information added
* faster rcnn variant and vehicle detector changes were made in 2021.1 and not in 2021.2
* some changes to support unit tests
* disable some tests which are failing
* fix myriad tests for vehicle detector
* Did some cleanup
*cleaned up comments
*Disabled Add_Broadcast_0x1 and Add_Broadcast_1x0
tests on MYRIAD_FP16 backend due to a bug
*cleaned up capability_2021_2.cc file
*Removed extra conditions which were added
for some validation in backend_utils
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* yolov3 pytorch workaround to ensure that the output names are matched
* gemmoptest fixed on myriad
* Fixed MYRIADX CPP Test Failures
*Expand,GatherND,Range,Round op's
are only supported in model
*where op with float input data
types are not supported and fixed
*Scatter and ScatterElements op's with
negative axis are fixed
*Reshape op with 0 dim value are not
supported and fixed
*Disabled InstanceNorm_2 test on MYRIADX
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* make changes to yolov3 pytorch
* Fixed python unit tests
*Fixed failing python tests on vpu,
GPU and CPU
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Fixes POW op failures on GPU_FP16
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Clean up capability_2021_2.cc
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Updated docx for MultiThreading option
*Added extra info on setting the num_of_threads
option using the API and it's actual usage
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* fixed slice and removed extra prints
* Disabled failing python tests
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Minor changes added in capabilty_2021_2
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* made changes to slice to avoid failures
* Disabling FP16 support for GPU_FP32
->Inferencing an FP16 model on GPU_FP32
leads to accuracy mismatches. so, we would
rather use GPU_FP16 to infer an FP16 model
on GPU Device
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Updated docx for Inferencing a FP16 Model
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* fix for mask rcnn
* Script for installing openvino from source
* Updated with openvino 2021.2 online installation
* code comment fixes
fixed accuracy mismatch for div
* Update OpenvinoEP-ExecutionProvider.md
updated for 2021.2 branch
* Update README.md
updated dockerfile documentation
* Update BUILD.md
build.md update documentation
* permissiong change of install_openvino.sh
* made changes to align with microsoft onnxruntime changes
* Updated with ov 2021.2.200
Co-authored-by: suryasidd <surya.siddharth.pemmaraju@intel.com>
Co-authored-by: sfatimar <sahar.fatima@intel/com>
Co-authored-by: MaajidKhan <n.maajidkhan@gmail.com>
Co-authored-by: mohdansx <mohdx.ansari@intel.com>
* Fix a memory leak in test_inference.cc (#6201)
* Fix a memory leak in test_inference.cc
* Use TArray in AMD element-wise kernels, rather than manually copying memory to device.
* Remove most ROCm-specific element-wise code and reuse CUDA element-wise code.
* Minor change to improve performance for operator Pad. (#5537)
* small improvment for pad
* Support double for operators Log, Reciprocal, Sum (CPU) (#6032)
* Support double for operators Log, Reciprocal, Sum
* remove tesdt erf_double
* Support double for operators Where, LpNormalisation (#6034)
* Support double for operators Relu, Tanh, Sigmoid (#6221)
* Fix ImportError in build.py (#6231)
There is a possible ImportError where build.py can import the wrong 'util' package if there are others present in `sys.path` already
* Removed executor todo that looks dead. (#6234)
* Remove MKLML/openblas/jemalloc build config (#6212)
* Remove python 3.5
* Update the readme file
* Upgrade build.py to assert for python 3.6+
Upgrade build.py to assert for python 3.6+
as python 3.5 cannot build anymore todays master.
* Support MLFloat16 type in Pow opset-12 CUDA kernel (#6233)
* MLAS: handle MlasGemm(M/N/K==0) cases (#6238)
* Support double for operator TopK + fix one bug in TopK implementation for GPU for double (#6220)
* Support double for operator TopK
* add static classes for topk/double
* fix cast issue in topk
* Support double for operator Gemm + fix bug in gemm implementation for cuda, rocm when sizeof(type) != sizeof(float) (#6223)
* Support double for operator Gemm
* fix type size while copying data in gemm operator for GPU
* fix type in gemm implementation for rocm
* Support double for operator ReduceMean, ReduceLogSumExp (#6217)
* Support double for operators ReduceMean, ReduceLogSumExp
* Support double for operator ArgMin (#6222)
* Support double for operator ArgMin
* add test specifically for double
* add new test on pai-excluded-tests.txt
* Update BUILD.md
* Update manylinux docker image to the latest (#6242)
* Fix allocator issue for TensorRT IOBinding (#6240)
* Fix issue: https://github.com/microsoft/onnxruntime/issues/6094
Root cause: we didn't expose the OrtMemoryInfo for TRT, so it will cause issue if user want use IObinding for Tensorrt.
Short term fix, add the OrtMemoryInfo for TRT. Long term should unify the allocator for CUDA and TRT
* Tune BiasGeluGradDx kernel in approximation mode to avoid tanh(...) on Rocm (#6239)
* bias gelu grad use exp(...) instead
* update cuda to rocm
* missing semicolon
* comment
* remove dockerfile
* missing factor of two
* Refactor EP Perf Tool (#6202)
* merge master, keep postprocess status commit
* download float16.py everytime
* using variables to reference eps
* adding ACL EP to ep perf tool
* accuracy with absolute tolerance configurable
* add acl to dict + remove commented line
* Documentation for distributed CI tests pipeline (#6140)
* Remove a debug log in provider_test_utils.cc (#6200)
* Add the Concat Slice Elimination transform, fix constant_folding transform (#5457)
* Add concat slice transform + test
* Cosmetic improvements in concat slice transform
* Remove unrelated file, fix comment, fix constant folding bug
* Add test onnx graph
* fix windows build
* Review comments
* review comment
Co-authored-by: Aishwarya <aibhanda@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add MakeStringLite which uses current locale, update some MakeString call sites to use it instead. (#6252)
* Add MakeStringLite which uses current locale, update macros to use that to generate messages.
* Convert calls to MakeStringLite().
* Liqun/speech model loop to scan (#6070)
Provide a tool to convert Loop to Scan for Nuphar performance
Fix Nuphar CI pipeline failures.
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* model parallel refinement (#6244)
* Megatron Transformation as a seperate step
* remove useless header
* clang formating
* Re-Structure megatron transformer for subsquent changes
* fix comments
* Allow querying a GraphProto's doc_string as part of ModelMetadata (#6248)
* Fix Linux/Mac error message on input type mismatch (#6256)
* add bfloat16 to gathergrad type constrains (#6267)
Co-authored-by: Cheng Tang <chenta@microsoft.com>
* Fix VS 2017 build break (#6276)
* Deprecate Python global configuration functions [Part 2] (#6171)
Update Python API to allow more flexibility for setting providers and provider options.
The providers argument (InferenceSession/TrainingSession constructors, InferenceSession.set_providers()) now also accepts a tuple of (name, options dict).
Fix get_available_providers() API (and the corresponding function in the C API) to return the providers in default priority order. Now it can be used as a starting point for the providers argument and maintain the default priority order.
Convert some usages of the deprecated global configuration functions to use EP-specific options instead.
Update some EP-specific option parsing to fail on unknown options.
Other clean up.
* Add script to preprocess python documentation before publishing (#6129)
* add script to preprocessing python documentation before publishing
* rename past to past_key_values for GPT-2 (#6269)
rename past to past_key_values for transformers 4.*
* Rename MakeString and ParseString functions. (#6272)
Rename MakeString to MakeStringWithClassicLocale, MakeStringLite to MakeString, *ParseString to *ParseStringWithClassicLocale.
Add missing pass-through versions of MakeStringWithClassicLocale for string types.
* Increase timeout for Linux GPU CUDA11 build. (#6280)
* Add helper to compare model with different precision (#6270)
* add parity_check_helper.py
* add real example
* remove lines
* Fix Min/Max CPU kernels for float16 type (#6205)
* fix data_ptr assertion error for past_sequence_length=0 in GPT-2 (#6284)
fix io binding crash for past_sequence_length=0
* A list of changes in transformers tool (#6224)
* longformer fp16 e2e
* add fp16/fp32 parity check helper file
* excludes nodes with subgraph in profiling
* use onnxconverter_common to do fp32->fp16
* add version check for onnxconverter_common
* remove helper file
* add pkg installation on notebooks and script
* Workaround for static_cast<double>(half)
* Add workaround to remove ROCm-specific binary-elementwise files.
* Update nuget build (#6297)
1. Update the ProtoSrc path. The old one is not used anymore.
2. Regenerate OnnxMl.cs
3. Delete some unused code in tools/ci_build/build.py
4. Avoid set intra_op_param.thread_pool_size in ModelTests in OpenMP build.
5. Fix a typo in the C API pipeline.
* Enable ONNX backend test of SequenceProto input/output (#6043)
* assert sequence tensor and remove skips
* update testdata json
* use ONNX 1.8 in cgmanifest.json
* use previous commit to workaround
* update ONNX commit ID in docker
* skip test_maxpool_2d_dilations test for now
* update function name
* add --sequence_lengths option (#6285)
* more dtype for Equal CUDA kernel (#6288)
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
* Force reinstall onnx python package on Windows (#6309)
* update transformers required package versions (#6315)
* Remove abs in LpPool (#6303)
* Support 1D input for Conv + Mul/Add fusion optimizer with test (#6295)
* Support 1D input (N C H) for Conv + Mul/Add fusion optimizer with test cases and test models.
* Add longformer to python package (#6314)
* add longformer to python package
* move test related script and data to a new folder
* Avoid false sharing on thread pool data structures (#6298)
Description: This change adds alignment and padding to avoid false sharing on fields in the thread pool. It also adds a new microbenchmark to profile thread-pool performance over short loops.
Motivation and Context
MobileNet on a 2*12-core system showed a performance gap between the ORT thread pool and OpenMP. One cause appeared to be false sharing on fields in the thread pool: ThreadPoolParallelSection::tasks_finished (which the main thread spins on waiting for workers to complete a loop), and the RunQueue::front_ and back_ fields (used respectively by the worker thread and the main thread).
The additional micro-benchmark BM_ThreadPoolSimpleParallelFor tests performance of loops of different sizes at different thread counts. The results below are on a machine with 2*14-core processors (E5-2690 v4) running with 1, 14, 15, and 28 threads. For each test, the microbenchmark has N threads run a loop with N iterations; hence a perfect result is for the time taken to be constant as additional threads are added (although we will also see power management effects helping at very low thread counts). The loop durations (100000, 10000, 1000) correspond roughly to 200us, 20us, and 2us on this machine.
Before change:
BM_ThreadPoolSimpleParallelFor/1/1/100000/real_time 17153 us 17154 us 32
BM_ThreadPoolSimpleParallelFor/14/14/100000/real_time 22553 us 22553 us 30
BM_ThreadPoolSimpleParallelFor/15/15/100000/real_time 21521 us 21521 us 29
BM_ThreadPoolSimpleParallelFor/28/28/100000/real_time 24111 us 24111 us 24
BM_ThreadPoolSimpleParallelFor/1/1/10000/real_time 1719 us 1719 us 407
BM_ThreadPoolSimpleParallelFor/14/14/10000/real_time 3409 us 3409 us 200
BM_ThreadPoolSimpleParallelFor/15/15/10000/real_time 3541 us 3541 us 201
BM_ThreadPoolSimpleParallelFor/28/28/10000/real_time 4576 us 4576 us 151
BM_ThreadPoolSimpleParallelFor/1/1/1000/real_time 174 us 174 us 4017
BM_ThreadPoolSimpleParallelFor/14/14/1000/real_time 1586 us 1586 us 402
BM_ThreadPoolSimpleParallelFor/15/15/1000/real_time 1586 us 1586 us 397
BM_ThreadPoolSimpleParallelFor/28/28/1000/real_time 2864 us 2864 us 232
After change:
BM_ThreadPoolSimpleParallelFor/1/1/100000/real_time 17160 us 17160 us 33
BM_ThreadPoolSimpleParallelFor/14/14/100000/real_time 20989 us 20989 us 31
BM_ThreadPoolSimpleParallelFor/15/15/100000/real_time 22286 us 22286 us 31
BM_ThreadPoolSimpleParallelFor/28/28/100000/real_time 24631 us 24631 us 25
BM_ThreadPoolSimpleParallelFor/1/1/10000/real_time 1718 us 1718 us 407
BM_ThreadPoolSimpleParallelFor/14/14/10000/real_time 2868 us 2868 us 242
BM_ThreadPoolSimpleParallelFor/15/15/10000/real_time 2907 us 2907 us 240
BM_ThreadPoolSimpleParallelFor/28/28/10000/real_time 3872 us 3872 us 186
BM_ThreadPoolSimpleParallelFor/1/1/1000/real_time 175 us 175 us 3938
BM_ThreadPoolSimpleParallelFor/14/14/1000/real_time 933 us 933 us 659
BM_ThreadPoolSimpleParallelFor/15/15/1000/real_time 912 us 912 us 591
BM_ThreadPoolSimpleParallelFor/28/28/1000/real_time 1976 us 1976 us 317
* fix opset imports for function body (#6287)
* fix function opsets
* add tests and update onnx
* changes per review comments
* add comments
* plus updates
* build fix
* Remove false positive prefast warning from threadpool (#6324)
* Java: add Semmle to Java publishing pipelines (#6326)
Add Semmle to Java API pipeline
Add security results publishing and add Java GPU.
* Quantization support for split operator with its NHWC support (#6107)
* Make split working for quantization.
* NHWC transformer support for split operator
* Refactor some according to Feedback. Will add test cases soon.
* Fix build error on windows.
* Add test case for split op on uint8_t support
* Add nhwc_transformer_test for split uint8_t support
* Some change according to PR feedbacks.
* Liqun/enable pipeline parallel test (#6331)
enable pipeline parallel test
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Use onnxruntime_USE_FULL_PROTOBUF=OFF for the cuda execution provider (#6340)
This removes a special case of the cuda EP.
* MLAS: add fallback implementation for quantized GEMM (#6335)
Add a non-vectorized version of the kernel used for the quantized version of MlasGemm.
* Delete float16.py (#6336)
No longer needed. Also doesn't pass policheck.
* Enable add + softmax fusion for Rocm platform (#6259)
* add bias softmax; tests appear to pass
* check fusion occurs for rocm as well
* check for rocm provider compatible as well
* build for cpu scenario as well
* try again; broader cope
* proper scope on kGpuExecutionProvider
* been editing wrong file
* remove commented #include lines
* try again due to mac os ci error
* try again
* test fusion both cuda and rocm to avoid mac ci error
* add external data support to tensor proto utils (#6257)
* update unpack tensor utilities to support loading external data
* more updates
* fix test
* fix nuphar build
* minor build fix
* add tests
* fix Android CI
* fix warning
* fix DML build failure and some warnings
* more updates
* more updates
* plus few updates
* plus some refactoring
* changes per review
* plus some change
* remove temp code
* plus updates to safeint usage
* build fix
* fix for safeint
* changed wording. (#6337)
* Remove OpSchema dummy definition. Only needed for Function now, and we can just exclude the method in Function (#6321)
* remove gemmlowp submodule (#6341)
* [NNAPI] Add pow support (#6310)
* Add support for running Android emulator from build.py on Windows. (#6317)
* fix the pipeline failure (#6346)
* Train BERT Using BFloat16 on A100 (#6090)
* traing bert using bf16
* Adam support bf16
* bugfix
* add fusedmatmul support
* fix after merge from master.
* bugfix
* bugfix after merge from master
* fast reduction for bf16.
* resolve comments
* fix win build
* bugfix
* change header file.
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
* Fix DerefNullPtr issues raised by SDLNativeRules. (#6348)
* update quantize to support basic optimization and e2e example for image classification (#6313)
update the resnet50-v1 to standard one from onnx zoo.
add an example for mobilenet
run basic optimization before quantization
fix a bug in Clip
* Enable graph save for orttrainer (#6333)
* Enable graph save for orttrainer
* Fix CI
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add PREfast to python packaging pipeline (#6343)
* Add PREfast to python packaging pipeline
* fix longformer benchmark io_binding output_buffers (#6345)
* fix longformer benchmark io_binding output_buffers
* format
* import benchmark_helper from parent directory.
* Use readelf for minimal build binary size checks. (#6338)
* Use readelf for minimal build binary size checks.
The on-disk size grows in 4KB chunks which makes it hard to see how much growth an individual checkin causes.
Only downside is that the sum of the sections is larger than the on-disk size (assumably things get packed smaller on disk and some of the section alignment constraints can be ignored)
* Remove unused function
* Java: Set C language warnings to W4 and adjust JNI code (#6347)
Set /W3 for C language and fix up JNI warnings.
* Pipeline Parallel Experimental Python API (#5815)
* Add create session to WinML telemetry to track WinML Usage (#6356)
* Fix one more SDL warning (#6359)
* fix -Wdangling-gsl (#6357)
* Add python example of TensorRT INT8 inference on ResNet model (#6255)
* add trt int8 example on resnet model
* Update e2e_tensorrt_resnet_example.py
* remove keras dependency and update class names
* move ImageNetDataReader and ImageClassificationEvaluator to tensorrt resnet example
* simplify e2e_tensorrt_resnet_example.py
* Update preprocessing.py
* merge tensorrt_calibrate
* Update calibrate.py
* Update calibrate.py
* generalize calibrate
* Update calibrate.py
* fix issues
* fix formating
* remove augment_all
* This added telemetry isn't needed (#6363)
* Wezuo/memory analysis (#5658)
* merged alloc_plan
* pass compilation
* Start running, incorrect allocation memory info
* add in comments
* fix a bug of recording pattern too early.
* debugging lifetime
* fix lifetime
* passed mnist
* in process of visualization
* Add code to generate chrome trace for allocations.
* in process of collecting fragmentation
* before rebuild
* passed mnist
* passed bert tiny
* fix the inplace reuse
* fix the exception of weight in pinned memory
* add guards to ensure the tensor is in AllocPlan
* add customized profiling
* debugging
* debugging
* fix the reuse of differnt location type
* add rank
* add the rank
* add fragmentation
* add time_step_trace
* Add summary for each execution step (total bytes, used/free bytes).
* add top k
* change type of top k parameter
* remove prints
* change heap to set{
* add the name pattern
* add the useage for pattern
* add partition
* change to static class
* add custom group
* remove const
* update memory_info
* in process of adding it as runtime config
* change the memory profiling to be an argument
* add some comments
* add checks to recored meomry_info in traaining session
* set the "local rank setting" to correct argument.
* addressing comments
* format adjustment
* formatting
* remove alloc_interval
* update memory_info.cc to skip session when there is no tensor for a particular memory type
* fix memory_info multiple iteration seg-fault
* consolidate mainz changes
* fixed some minor errors
* guard by ORT_MINIMAL_BUILD
* add ORT_MEMORY_PROFILE flag
* added compiler flag to turn on/off memory profiling related code
* clean up the code regarding comments
* add comments
* revoke the onnx version
* clean up the code to match master
* clean up the code to match master
* clean up the code to match master
Co-authored-by: Jesse Benson <benson.jesse@gmail.com>
Co-authored-by: Wei Zuo <wezuo@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-mgtbby.eastus.cloudapp.azure.com>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-yclzsf.eastus.cloudapp.azure.com>
* Support MLFloat16 in CumSum Cuda op for Opset 14 (#6355)
* Add CumSum-14 for Cuda
* fix convert_common version retrival (#6382)
* Refine auto_pad based pad computation in ConvTranspose (#6305)
* Fix SDL warning (#6390)
* Add max_norm for gradient clipping. (#6289)
* add max_norm as user option for gradient clipping
* add adam and lamb test cases for clip norm
* add frontend tests
* Add the custom op project information (#6334)
* Dont use default string marshalling in C# (#6219)
* Fix Windows x86 compiler warnings in the optimizers project (#6377)
* [Perf] Optimize Tile CPU and CUDA kernels for a corner case (#6376)
* Unblock Android CI code coverage failure (#6393)
* fix build on cuda11 (#6394)
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
* Load the model path correctly (#6369)
* Fix some compile warnings (#6316)
* OpenVino docker file changes to bypass privileged mode
Description: Builds and installs libusb without UDEV support, which is used for communicating with the VPU device.
Motivation and Context
This enables the resulting docker container to be run without '--privileged' and '--network host' options which may not be suitable in deployment environments.
* Megatron checkpointing (#6293)
* Add bart fairseq run script
* Add frontend change to enable megatron
* Initial changes for checkpointing
* Megatron optim state loading, checkpoint aggregation, frontend distributed tests for H, D+H
* Add load_checkpoint changes
* Fix CI
* Cleanup
* Fix CI
* review comments
* review comments
* review comments:
* Fix generate_submodule_cgmanifest.py Windows issues. (#6404)
* Continue memory planning when unknown shape tensor is encountered. (#6413)
* Reintroduce experimental api changes and fix remote build break (#6385)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* Add support for custom ops to minimal build. (#6228)
* Add support for custom ops to minimal build.
Cost is only ~8KB so including in base minimal build.
* enable pipeline to run quantization tests (#6416)
* enable pipeline to run quantization tests
setup test pipeline for quantization
* Minor cmake change (#6431)
* Liqun/liqun/enable pipeline parallel test2 (#6399)
* enable data and pipeline parallism test
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Farewell TrainableDropout (#5793)
* Deprecate TrainableDropout kernel.
* Update bert_toy_postprocessed.onnx to opset 12.
* Add more dropout tests.
* Fix BiasDropout kernel.
Co-authored-by: Ubuntu <OrtTrainingDev3@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sherlock Huang <bahuang@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sergii Dymchenko <sedymche@microsoft.com>
* fix null dereference warning (#6437)
* Expose graph ModelPath to TensorRT shared library (#6353)
* Update graph_viewer.cc
* Update tensorrt_execution_provider.cc
* Update graph_viewer.h
* Update tensorrt_execution_provider.cc
* Update tensorrt_execution_provider.cc
* Update provider_api.h
* Update provider_bridge_ort.cc
* Update provider_interfaces.h
* Update provider_interfaces.h
* expose GraphViewer ModelPath API to TRT shared lib
* add modelpath to compile
* update
* add model_path to onnx tensorrt parser
* use GenerateMetaDefId to generate unique TRT kernel name
* use GenerateMetaDefId to generate unique TRT engine name
* fix issue
* Update tensorrt_execution_provider.cc
* remove GetVecHash
* Update tensorrt_execution_provider.h
* convert wchar_t to char for tensorrt parser
* update tensorrt parser to include latest changes
* fix issues
* Update tensorrt_execution_provider.cc
* merge trt parser latest change
* add PROVIDER_DISALLOW_ALL(Path)
* add tool for generating test data for longformer (#6415)
* only build experimental api in redist (#6465)
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
* Add an option to save the training graph after optimization (#6410)
* expose optimized_model_filepath in SessionOptions as `debug.graph_save_paths.model_with_training_graph_after_optimization_path` in `ORTTrainerOptions`
* Share allocator between CUDA EP & TRT EP. (#6332)
* Share allocator between CUDA EP & TRT EP.
limitation:
1. Does not cover the per-thread allocator created by CUDA EP, still need to figure out the way to remove it
2. Need to have more identifiers to make it able to share CPU allocator across all EPs
* fix max norm clipping test in python packaging pipeline test (#6468)
* fix python packaging pipeline
* make clip norm test compatabile with both V100 and M60 GPUs
* Initial version of CoreML EP (#6392)
* Bug 31463811: Servicing: Redist (Nuget) conflicts with Microsoft.AI.MachineLearning starting 21H1+ (#6460)
* update load library code to have the fullly qualified path
* make it work for syswow32
* git Revert "make it work for syswow32"
This reverts commit b9f594341b7cf07241b18d0c376af905edcabae3.
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
* dequantize 1st input of lstm back if it is quantized (#6444)
* [java] Adds support for OrtEnvironment thread pools (#6406)
* Updates for Gradle 7.
* Adding support for OrtThreadingOptions into the Java API.
* Fixing a typo in the JNI code.
* Adding a test for the environment's thread pool.
* Fix cuda test, add comment to failure.
* Updating build.gradle
* fix SDL native rule warning #6246 (#6461)
* fix SDL rule (#6464)
* use tickcount64 (#6447)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* Update pypi package metadata (#6354)
* Update setup file data
* add missing comma
* remove python 3.5
* fix typo bracket
* Delete nuget extra configs (#6477)
* Op kernel type reduction infrastructure. (#6466)
Add infrastructure to support type reduction in Op kernel implementations.
Update Cast and IsInf CPU kernels to use it.
* Fixing a leak in OnnxSequences with String keys or values. (#6473)
* Increase the distributes tests pipeline timeout to 120 minutes (#6479)
* [CoreML EP] Add CI for CoreML EP (macOS) and add coreml_flags for EP options (#6481)
* Add macos coreml CI and coreml_flags
* Move save debuggubg model to use environment var
* Move pipeline off from macos CI template
* Fix an issue building using unix make, add parallel to build script
* Fixed build break for shared_lib and cmpile warning
* Fix a compile warning
* test
* Revert the accidental push from another branch
This reverts commit 472029ba25d50f9508474c9eeceb3454cead7877.
* Add ability to track per operator types in reduced build config. (#6428)
* Add ability to generate configuration that includes required types for individual operators, to allow build size reduction based on that.
- Add python bindings for ORT format models
- Add script to update bindings and help info
- Add parsing of ORT format models
- Add ability to enable type reduction to config generation
- Update build.py to only allow operator/type reduction via config
- simpler to require config to be generated first
- can't mix a type aware (ORT format model only) and non-type aware config as that may result in insufficient types being enabled
- Add script to create reduced build config
- Update CIs
* merge e2e with distributed pipeline (#6443)
merge e2e with distributed pipeline
* Fix test breaks in Windows ingestion pipeline (#6476)
* fix various build breaks with Windows build
* fix runtime errors loading libraries from system32
* add build_inbox check to winml_test_common
* use raw string
* cleanup
* fix dll load
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
* Speed up the Mac CI runs (#6483)
* expose learningmodelpixelrange property (#5877)
* Fix of support api version bug for [de]quantize (#6492)
* SDL fixes: add proper casts/format specifiers (#6446)
* SDL annotation fixes (#6448)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* [OpenVINO-EP] Remove support for OpenVINO 2020.2 (#6493)
* Removed OpenVINO 2020.2 support
* Updated documentation and build.py
* Removed unnecessary libraries from setup.py
* Support pad operator in quantization and quantized nhwc transformer. Fix Pad operator bug. (#6325)
Support pad operator in quantization tool.
Support pad operator in quantized nhwc transformer.
Fix pad() operator bug when pad input's inner(right) most axis value is zero for Edge and Reflect mode, it copied wrong value to the cells to be padded. Note the Constant mode will not trigger this bug, as Edge/Reflect need copy value from the already copied array while Constant mode only fill specified value.
Add more test cases to cover pad() operator bug fixed here.
Fix quantization tools uint8/int8 value overflow issue when quantize weights in python.
* Improve work distribution for Expand operator, and sharded LoopCounter configuration (#6454)
Description: This PR makes two changes identified while looking at a PGAN model.
First, it uses ThreadPool::TryParallelFor for the main parallel loops in the Expand operator. This lets the thread pool decide on the granularity at which to distribute work (unlike TrySimpleParallelFor). Profiling showed high costs when running "simple" loops with 4M iterations each of which copied only 4 bytes.
Second, it updates the sharded loop counter in the thread pool so that the number of shards is capped by the number of threads. This helps make the performance of any other high-contention "simple" loops more robust at low thread counts by letting each thread work on its own "home" shard for longer.
Motivation and Context
Profiling showed a PGAN model taking 2x+ longer with the non-OpenMP build. The root cause was that the OpenMP build uses simple static scheduling of loop iterations, while the non-OpenMP build uses dynamic scheduling. The combination of large numbers of tiny iterations is less significant with static scheduling --- although still desirable to avoid, given that each iteration incurs a std::function invocation.
* Update document of transformer optimization (#6487)
* nuphar test to avoid test data download to improve passing rate (#6467)
nuphar test to avoid test data download to improve passing rate
* Fuse cuda conv with activation (#6351)
* optimize cuda conv by fused activation
* remove needless print out
* exclude test from cpu
* handle status error from cudnn 8.x
* add reference to base class
* add hipify
* [CoreML EP] Add support for some activations/Transpose, move some shared helpers from NNAPI to shared space (#6498)
* Init change
* Move some helper from nnapi ep to shared
* Add transpose support
* Fix trt ci build break
* Refine transformers profiler output (#6502)
* output nodes in the original order; grouped by node name
* add document for profiler
* Update to match new test setup. (#6496)
* Update to match new test setup.
* Add Gemm(7) manually for now.
Will fix properly on Monday. It's used by mnist.ort as that is created by optimizing mnist.onnx to level 1 causing 2 nodes to be replaced by a Gemm and the op to be missing from the required list as that is created using the original onnx model.
* Enable dense sequence optimized version of Pytorch exported BERT-L on AMD GPU (#6504)
* Permit dense seq optimization on BERT-L pytorch export by enabling ReduceSumTraining, Equal, and NonZero on AMD
* enable Equal tests
* enable fast_matrix_reduction test case
* Optimize GatherGrad for AMD GPU (#6381)
* optimize gathergrad
* address comments
Co-authored-by: Weixing Zhang <wezhan@microsoft.com>
* add explicit barriers for buffer overread and overrwrite (#6484)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* fix sdl bugs for uninitialized variables and returns (#6450)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* handle hr error conditions (#6449)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* Dnnl training (#6045)
* Add ReluGrad and ConvGrad ops for the dnnl provider
* the mnist sample is updated to add the --use_dnnl option that
will cause the sample to use the dnnl execution provider for
nodes that exist in dnnl provider.
* Added the ability to find forward ops. Dnnl backward gradient
ops require the forward primitive description and workspace
from the forward operation.
* Enable specifying the execution provider for Gradient Checker Tests
* Prevent memory leak when running dnnl_provider in training mode
Prevent creating a SubgraphPrimitivePool when the code is built with the
ENABLE_TRAINING build flag. Instead create a SubgraphPrimitive directly.
The SubgraphPrimitivePool was causing a pool of SubgraphPrimitives to be
stashed in a map for reuse. Due to the way the Training Loop uses threads
the pool of SubgraphPrimitives were not being reuse instead a new pool of
SubgraphPrimitives being created each run. The old pool was not instantly
freed. This behavior could be a language error when using thread_local
memory.
Signed-off-by: George Nash <george.nash@intel.com>
* Added fixes to maxpoolgrad and memory leak.
Maxpoolgrad will now pass all unit tests.
With the conv and convgrad disabled for dnnl, mnist is able to train till 95%
Signed-off-by: Chethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
* Fixed misc issues when testing training code with dnnl provider
* fix conv_grad dnnl tests with dilation to run dnnl execution provider
* update mnist training sample to accept convolution type models
convolution models require the input shape to be {1, 28, 28}
instead of the flat {728} image that is used for the gemm models
this will enable models that require the different shape by adding
`--model_type conv` to the command line when running the mnist sample.
(while testing a workaround was used see #4762)
* Disable weight caching in dnnl conv operator when using training
When training we can not use cached weights because the weight
will be updated each run. This re-enables dnnl Conv and ConvGrad Ops.
The weight caching was the source of the error from Conv when training.
* Fix issues found when building grad ops on Linux
* The dnnl_convgrad code was over using the scope operator
causing a compilation problem.
* The dnnl_maxpoolgrad code had a logic error that is was
comparing with the source description when it should have
been comparing with the destination despription.
* Update BUILD.md so it shows DNNL for training
* Updated the table of contents. Since the same providers
are listed twice. Once for Infrance and again for Training
an HTML anchor was added to distinguish the second header
from the first for the TOC.
* Fix build failure when not using --enable-training build option
* reorganize the gradient operators so they are grouped together
* Fix issues found when running onnx_backend_test_series.py
* Pooling code only supports 2 outputs when built with --enable-training
* Address code review feedback
* class member variables end in underscore_
* use dst instead of dist to match pattern use elsewhere in DNNL code.
* Remove workaround that was introduced to handle problems running
convolution based training models. See issue #4762
Signed-off-by: George Nash <george.nash@intel.com>
* Isolate training code and code cleanup
* Do not build if dnnl_gpu_runtime if enable_training is set training code
does not support dnnl_gpu_runtime yet.
* Isolated Training code inside ifdefs so that they wont affect
project if built without training enabled
* Inadvertant changes in whitespace were removed to make code review simpler
* Undid some code reordering that was not needed
* comments added to closing #endif statments to simplify reading complex ifdefs
* Modified the GetPrimitiveDesc functions to return shared_ptr instead of raw
pointer. This matches what was done in Pool code and is safer memory code.
Signed-off-by: George Nash <george.nash@intel.com>
* Address code review issues
- whitespace changes caused by running clang-format on the code
- Several spelling errors fixed
- Removed/changed some ifdefs to improve readability
- other misc. changes in responce to code review.
Signed-off-by: George Nash <george.nash@intel.com>
* Code changes to address code review
- Simplify iteration code using `auto` keyword
- remove C style cast that was not needed
- remove instance variable that was not needed [relugrad.h]
- added the execution providers to `ComputeGradientErrorInternal()`
and `ComputeTheoreticalJacobianTranspose()` instead of using
a pointer to an instance varaible [gradient_checker.h/.cc]
Signed-off-by: George Nash <george.nash@intel.com>
* Combined the default gradient ops test and dnnl gradient ops test for ConvGrad and MaxPoolGrad into one function with the help of a helper function.
This will reduce repeated code.
Signed-off-by: Palangotu Keshava, Chethan's avatarChethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
* Replaced the stack used by convgrad to vector so that the vector(used as stack) can be easily cleared everytime the graph is created.
This will prevent memory leak from convolution kernels being pushed constantly onto the stack.
Signed-off-by: chethan.palangotu.keshava@intel.com
* Code clean up and formating updates
- Removed empty else statment
- updated indentation of code that was causing double curly brackets to look unususal
- Changed check for NumDimensions to Size in Relu and ReluGrad error checking code.
- isolated training code
Signed-off-by: George Nash <george.nash@intel.com>
* Restore inadvertantly removed ConvGrad tests
When combining the DNNL and CPU version of the ConvGrad
tests two test were inadvertantly excluded. This adds
back the Conv3d and Conv3d with strides test cases.
Signed-off-by: George Nash <george.nash@intel.com>
* Add validation to ConvGrad
This validates the dimensions of the ConvGrad match the
passed in Convolution forward primitive description.
The current code for DNNL ConvGrad makes the assumption that the ConvGrad
nodes will be visited in the reverse order from the corresponding Conv nodes
The added validation will return an error if this assumption is not true.
Signed-off-by: George Nash <george.nash@intel.com>
* Do not create new execution providers in provider_test_utils
This removes the code that generated new execution providers in the
OpTester::Run function. This was added because the std::move was
leaving the `entry` value empty so subsequent calls would cause a
segfault.
Problem is this potentially changed the execution_provider because it
would create the default provider dropping any custom arguments.
When the now removed code was originally added the std::move was causing
crashes when the GradientChecker unit tests were run. However, it is no
longer causing problems even with the code removed.
Signed-off-by: George Nash <george.nash@intel.com>
* Change the forward conv stack to a forward conv map
This changes how the forward conv kernel is mapped to the bwd ConvGrad
kernel the problematic stack is no longer used.
The convolution stack made the assumption that the corresponding
ConvGrad operator would be visited in reverse order of the forward
Conv operators. This was always problematic and was unlikely to
work for inception models.
Important changes:
- The weight_name is added to the ConvGrad dnnl_node making it
possible to use the weight_name as a lookup key to find the
Conv forward Kernel
- the `std::vector fwd_conv_stack_` has been replaced with a
`std::map fwd_conv_kernel_map_`
- Although it is not needed lock_guards were added when writing
to and reading from the fwd_conv_kernel_map_ as well as the
fwd_kernel_map_. These should always be accessed by a single
thread when preparing the dnnl subgraphs so the guard should not
be needed but its added just in case.
- Updated the comments ConvGrad.h code to no longer mention the
stack. The error check is not removed. It will be good to verify
there are no errors as we continue to test against more models.
Signed-off-by: George Nash <george.nash@intel.com>
Co-authored-by: Chethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
Co-authored-by: unknown <63478620+jeyblu@users.noreply.github.com>
* Lochi/refactor yolov3 quantization (#6290)
* Refactor the code and move data reader, preprocessing, evaluation to
E2E_example_mode
* Refactor the code.
Move data reader, preprocessing, evaluation to model specific example
under E2E_example_mode
* refactor code
* Move yolov3 example to specific folder and add additional pre/post
processing
* Print a warning message for using newer c_api header on old binary (#6507)
* Fix issues with ArmNN build setup (#6495)
* ArmNN build fixes
* Update BUILD.md to document that the ACL paths must be specified to build ArmNN
* Fix CUDA build error. We don't setup the link libraries correctly/consistently so improve that.
* Fix Windows CI builds by updating test scripts to work with numpy 1.20. (#6518)
* Update onnxruntime_test_python.py to work with numpy 1.20.
Some aliases are deprecated in favor of the built-in python types. See https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
np.array with bytes for entries and dtype of np.void no longer automatically pads. Change a test to adjust for that.
* Fix another test script
* Fix ORTModule branch for orttraining-* pipelines
* Update pytorch nightly version dependency
Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com>
Co-authored-by: George Wu <jywu@microsoft.com>
Co-authored-by: Cecilia Liu <ziyue.liu7@gmail.com>
Co-authored-by: Ryan Hill <38674843+RyanUnderhill@users.noreply.github.com>
Co-authored-by: George Nash <george.nash@intel.com>
Co-authored-by: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com>
Co-authored-by: Yateng Hong <toothache9010@gmail.com>
Co-authored-by: stevenlix <38092805+stevenlix@users.noreply.github.com>
Co-authored-by: Derek Murray <Derek.Murray@microsoft.com>
Co-authored-by: ashbhandare <ash.bhandare@gmail.com>
Co-authored-by: Scott McKay <skottmckay@gmail.com>
Co-authored-by: Changming Sun <chasun@microsoft.com>
Co-authored-by: Tracy Sharpe <42477615+tracysh@users.noreply.github.com>
Co-authored-by: Juliana Franco <jufranc@microsoft.com>
Co-authored-by: Pranav Sharma <prs@microsoft.com>
Co-authored-by: Tixxx <tix@microsoft.com>
Co-authored-by: Jay Rodge <jayrodge@live.com>
Co-authored-by: Du Li <duli1@microsoft.com>
Co-authored-by: Du Li <duli@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Yufeng Li <liyufeng1987@gmail.com>
Co-authored-by: baijumeswani <bmeswani@microsoft.com>
Co-authored-by: Sergii Dymchenko <sedymche@microsoft.com>
Co-authored-by: jingyanwangms <47403504+jingyanwangms@users.noreply.github.com>
Co-authored-by: satyajandhyala <satya.k.jandhyala@gmail.com>
Co-authored-by: S. Manohar Karlapalem <manohar.karlapalem@intel.com>
Co-authored-by: Weixing Zhang <weixingzhang@users.noreply.github.com>
Co-authored-by: Suffian Khan <sukha@microsoft.com>
Co-authored-by: Olivia Jain <oljain@microsoft.com>
Co-authored-by: Chi Lo <54722500+chilo-ms@users.noreply.github.com>
Co-authored-by: Hariharan Seshadri <shariharan91@gmail.com>
Co-authored-by: Ryan Lai <rylai@microsoft.com>
Co-authored-by: Jesse Benson <jesseb@microsoft.com>
Co-authored-by: sfatimar <64512376+sfatimar@users.noreply.github.com>
Co-authored-by: suryasidd <surya.siddharth.pemmaraju@intel.com>
Co-authored-by: sfatimar <sahar.fatima@intel/com>
Co-authored-by: MaajidKhan <n.maajidkhan@gmail.com>
Co-authored-by: mohdansx <mohdx.ansari@intel.com>
Co-authored-by: Xavier Dupré <xadupre@users.noreply.github.com>
Co-authored-by: Michael Goin <mgoin@vols.utk.edu>
Co-authored-by: Michael Giba <michaelgiba@gmail.com>
Co-authored-by: William Tambellini <wtambellini@sdl.com>
Co-authored-by: Hector Li <hecli@microsoft.com>
Co-authored-by: Aishwarya <aibhanda@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: liqunfu <liqfu@microsoft.com>
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: pengwa <pengwa@microsoft.com>
Co-authored-by: Tang, Cheng <souptc@gmail.com>
Co-authored-by: Cheng Tang <chenta@microsoft.com>
Co-authored-by: Tianlei Wu <tlwu@microsoft.com>
Co-authored-by: Ye Wang <52801275+wangyems@users.noreply.github.com>
Co-authored-by: Chun-Wei Chen <jacky82226@gmail.com>
Co-authored-by: Vincent Wang <wangwchpku@outlook.com>
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
Co-authored-by: Luyao Ren <375833274@qq.com>
Co-authored-by: Zhang Lei <zhang.huanning@hotmail.com>
Co-authored-by: Tim Harris <tiharr@microsoft.com>
Co-authored-by: Ashwini Khade <askhade@microsoft.com>
Co-authored-by: Dmitri Smirnov <yuslepukhin@users.noreply.github.com>
Co-authored-by: Alberto Magni <49027342+alberto-magni@users.noreply.github.com>
Co-authored-by: Wei-Sheng Chin <wschin@outlook.com>
Co-authored-by: wezuo <49965641+wezuo@users.noreply.github.com>
Co-authored-by: Jesse Benson <benson.jesse@gmail.com>
Co-authored-by: Wei Zuo <wezuo@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-mgtbby.eastus.cloudapp.azure.com>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-yclzsf.eastus.cloudapp.azure.com>
Co-authored-by: Wenbing Li <10278425+wenbingl@users.noreply.github.com>
Co-authored-by: Martin Man <supermt@gmail.com>
Co-authored-by: M. Zeeshan Siddiqui <mzs@microsoft.com>
Co-authored-by: Ori Levari <ori.levari@microsoft.com>
Co-authored-by: Ori Levari <orlevari@microsoft.com>
Co-authored-by: Ubuntu <OrtTrainingDev3@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sherlock Huang <bahuang@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sheil Kumar <smk2007@gmail.com>
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
Co-authored-by: Ryota Tomioka <ryoto@microsoft.com>
Co-authored-by: Adam Pocock <adam.pocock@oracle.com>
Co-authored-by: Yulong Wang <f.s@qq.com>
Co-authored-by: Faith Xu <faxu@microsoft.com>
Co-authored-by: Xiang Zhang <xianz@microsoft.com>
Co-authored-by: suryasidd <48925384+suryasidd@users.noreply.github.com>
Co-authored-by: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com>
Co-authored-by: Weixing Zhang <wezhan@microsoft.com>
Co-authored-by: Chethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
Co-authored-by: unknown <63478620+jeyblu@users.noreply.github.com>
2021-02-02 16:59:56 +00:00
)
2023-04-26 17:45:52 +00:00
file ( GLOB onnxruntime_python_transformers_models_whisper_src CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / t o o l s / t r a n s f o r m e r s / m o d e l s / w h i s p e r / * . p y "
)
2019-04-29 19:58:20 +00:00
file ( GLOB onnxruntime_python_datasets_srcs CONFIGURE_DEPENDS
2018-11-20 00:48:22 +00:00
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / d a t a s e t s / * . p y "
)
2019-04-29 19:58:20 +00:00
file ( GLOB onnxruntime_python_datasets_data CONFIGURE_DEPENDS
2018-11-20 00:48:22 +00:00
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / d a t a s e t s / * . p b "
" $ { O N N X R U N T I M E _ R O O T } / p y t h o n / d a t a s e t s / * . o n n x "
)
2022-02-17 21:35:25 +00:00
# ORT Mobile helpers to convert ONNX model to ORT format, analyze model for suitability in mobile scenarios,
# and assist with export from PyTorch.
set ( onnxruntime_mobile_util_srcs
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / c h e c k _ o n n x _ m o d e l _ m o b i l e _ u s a b i l i t y . p y
2021-04-30 04:23:54 +00:00
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / c o n v e r t _ o n n x _ m o d e l s _ t o _ o r t . p y
2022-03-14 23:50:41 +00:00
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / f i l e _ u t i l s . p y
2021-04-30 04:23:54 +00:00
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / l o g g e r . p y
2022-02-17 21:35:25 +00:00
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / m a k e _ d y n a m i c _ s h a p e _ f i x e d . p y
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / o n n x _ m o d e l _ u t i l s . p y
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / o p t i m i z e _ o n n x _ m o d e l . p y
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / p y t o r c h _ e x p o r t _ h e l p e r s . p y
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / r e d u c e d _ b u i l d _ c o n f i g _ p a r s e r . p y
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / u p d a t e _ o n n x _ o p s e t . p y
2021-04-30 04:23:54 +00:00
)
file ( GLOB onnxruntime_ort_format_model_srcs CONFIGURE_DEPENDS
2022-02-17 21:35:25 +00:00
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / o r t _ f o r m a t _ m o d e l / * . p y
)
file ( GLOB onnxruntime_mobile_helpers_srcs CONFIGURE_DEPENDS
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / m o b i l e _ h e l p e r s / * . p y
$ { R E P O _ R O O T } / t o o l s / c i _ b u i l d / g i t h u b / a n d r o i d / n n a p i _ s u p p o r t e d _ o p s . m d
2024-06-17 21:50:33 +00:00
$ { R E P O _ R O O T } / t o o l s / c i _ b u i l d / g i t h u b / a p p l e / c o r e m l _ s u p p o r t e d _ m l p r o g r a m _ o p s . m d
$ { R E P O _ R O O T } / t o o l s / c i _ b u i l d / g i t h u b / a p p l e / c o r e m l _ s u p p o r t e d _ n e u r a l n e t w o r k _ o p s . m d
2022-02-17 21:35:25 +00:00
)
2022-03-15 05:52:12 +00:00
file ( GLOB onnxruntime_qdq_helper_srcs CONFIGURE_DEPENDS
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / q d q _ h e l p e r s / * . p y
)
2021-04-30 04:23:54 +00:00
2023-06-17 02:47:09 +00:00
if ( onnxruntime_USE_OPENVINO )
file ( GLOB onnxruntime_python_openvino_python_srcs CONFIGURE_DEPENDS
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / a d d _ o p e n v i n o _ w i n _ l i b s . p y
)
endif ( )
2021-02-24 04:21:57 +00:00
set ( build_output_target onnxruntime_common )
2021-04-29 18:54:57 +00:00
if ( NOT onnxruntime_ENABLE_STATIC_ANALYSIS )
2018-11-20 00:48:22 +00:00
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
2021-02-24 04:21:57 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / b a c k e n d
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i / t r a i n i n g
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / d a t a s e t s
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s
2022-02-17 21:35:25 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s / m o b i l e _ h e l p e r s
2022-03-15 05:52:12 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s / q d q _ h e l p e r s
2021-04-30 04:23:54 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s / o r t _ f o r m a t _ m o d e l
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s / o r t _ f o r m a t _ m o d e l / o r t _ f l a t b u f f e r s _ p y
2021-02-24 04:21:57 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s
2022-04-10 05:35:14 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s
2023-02-07 15:49:15 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / b a r t
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / b e r t
2022-04-20 18:09:26 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / g p t 2
2023-08-23 01:05:11 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / l l a m a
2022-04-10 05:35:14 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / l o n g f o r m e r
2024-02-05 18:15:16 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / p h i 2
2024-09-18 21:31:59 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / s a m 2
2023-02-07 15:49:15 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / s t a b l e _ d i f f u s i o n
2022-04-10 05:35:14 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / t 5
2023-04-26 17:45:52 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / w h i s p e r
2021-02-24 04:21:57 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n / o p e r a t o r s
2021-03-19 08:09:11 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n / C a l T a b l e F l a t B u f f e r s
2023-12-12 16:43:04 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n / f u s i o n s
[Quantization] Tensor quant overrides and QNN EP quantization configuration (#18465)
### Description
#### 1. Adds `TensorQuantOverrides` extra option
Allows specifying a dictionary of tensor-level quantization overrides:
```
TensorQuantOverrides = dictionary :
Default is {}. Set tensor quantization overrides. The key is a tensor name and the value is a
list of dictionaries. For per-tensor quantization, the list contains a single dictionary. For
per-channel quantization, the list contains a dictionary for each channel in the tensor.
Each dictionary contains optional overrides with the following keys and values.
'quant_type' = QuantType : The tensor's quantization data type.
'scale' = Float : The scale value to use. Must also specify `zero_point` if set.
'zero_point' = Int : The zero-point value to use. Must also specify `scale` is set.
'symmetric' = Bool : If the tensor should use symmetric quantization. Invalid if also
set `scale` or `zero_point`.
'reduce_range' = Bool : If the quantization range should be reduced. Invalid if also
set `scale` or `zero_point`.
'rmax' = Float : Override the maximum real tensor value in calibration data.
Invalid if also set `scale` or `zero_point`.
'rmin' = Float : Override the minimum real tensor value in calibration data.
Invalid if also set `scale` or `zero_point`.
```
- All of the options are optional.
- Some combinations are invalid.
- Ex: `rmax` and `rmin` are unnecessary if the `zero_point` and `scale`
are also specified.
Example for per-tensor quantization overrides:
```Python3
extra_options = {
"TensorQuantOverrides": {
"SIG_OUT": [{"scale": 1.0, "zero_point": 127}],
"WGT": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
"BIAS": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
},
}
```
Example for per-channel quantization overrides (Conv weight and bias):
```Python3
extra_options = {
"TensorQuantOverrides": {
"WGT": [
{
"quant_type": quantization.QuantType.QUInt8,
"rmin": 0.0,
"rmax": 2.5,
"reduce_range": True,
},
{
"quant_type": quantization.QuantType.QUInt8,
"rmin": 0.2,
"rmax": 2.55,
"reduce_range": False,
},
],
"BIAS": [
{"zero_point": 0, "scale": 0.000621},
{"zero_point": 0, "scale": 0.23},
],
},
}
```
#### 2. Adds utilities to get the default QDQ configs for QNN EP
Added a `quantization.execution_providers.qnn.get_qnn_qdq_config` method
that inspects the model and returns suitable quantization
configurations.
Example usage:
```python3
from quantization import quantize, QuantType
from quantization.execution_providers.qnn import get_qnn_qdq_config
qnn_config = get_qnn_qdq_config(input_model_path,
data_reader,
activation_type=QuantType.QUInt16,
weight_type=QuantType.QUInt8)
quantize(input_model_path,
output_model_path,
qnn_config)
```
### Motivation and Context
Make it possible to create more QDQ models that run on QNN EP.
---------
Signed-off-by: adrianlizarraga <adlizarraga@microsoft.com>
2023-12-05 01:54:58 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n / e x e c u t i o n _ p r o v i d e r s
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n / e x e c u t i o n _ p r o v i d e r s / q n n
2021-02-24 04:21:57 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / q u a n t i z a t i o n
2021-06-09 02:43:59 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / t r a n s f o r m e r s
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / t r a n s f o r m e r s / t e s t _ d a t a / m o d e l s
Whisper Model Optimization (#15473)
### Description
This PR contains fusion-level and kernel-level optimizations for
[OpenAI's Whisper](https://github.com/openai/whisper).
Some of the added optimizations include:
- Pruning of duplicate/unnecessary inputs and outputs
- Fusion support for Whisper models with or without these inputs/outputs
(e.g. with these inputs/outputs if exporting with an older official
Optimum version, without these inputs/outputs if exporting with Optimum
from source)
- Attention fusions
- For Whisper's encoder and decoder
- Modified symbolic shape inference for present output when no past
input exists (for decoder)
- Multi-head attention fusions
- For Whisper's decoder and decoder with past
- Packed MatMul for the 3 MatMuls excluded in multi-head attention
fusion
- Attention kernel changes
- CPU:
- Different Q and KV sequence lengths
- Parallel memset for large sequence lengths
- Convert broadcast add after MatMul of Q and K (add_qk) to element-wise
add
- Separate present key-value output into present key and present value
(for multi-head attention spec)
- CUDA:
- Use memory efficient attention compute kernel with present state (for
decoder)
- Multi-head attention kernel changes
- CPU:
- Introduction of multi-head attention CPU kernel (previously did not
exist)
- Use AddBiasReshape instead of AddBiasTranspose when sequence length =
1 (for decoder with past)
- Different Q, K, V input shapes
- Pass past key and past value directly as key and value
- CUDA:
- Use memory efficient attention compute kernel with past and/or present
state (for decoder with past)
### Usage
To use the optimizations, run the ORT transformer optimizer script as
follows:
```
$ cd onnxruntime/onnxruntime/python/tools/transformers/
$ python3 optimizer.py --input <filename>.onnx --output <filename>.onnx --model_type bart --num_heads <number of attention heads, depends on the size of the whisper model used> --hidden_size <attention hidden size, depends on the size of the whisper model used> --use_external_data_format --use_multi_head_attention
```
Once optimized, here's an example of how to run Whisper with [Hugging
Face's Optimum](https://github.com/huggingface/optimum):
```
from transformers.onnx.utils import get_preprocessor
from optimum.onnxruntime import ORTModelForSpeechSeq2Seq
from optimum.pipelines import pipeline as ort_pipeline
import whisper # Installed from OpenAI's repo - setup instructions at https://github.com/openai/whisper/
directory = './whisper_opt' # Where the optimized ONNX models are located
model_name = 'openai/whisper-tiny'
device = 'cpu'
# Get pipeline
processor = get_preprocessor(model_name)
model = ORTModelForSpeechSeq2Seq.from_pretrained(
directory,
use_io_binding=(device == 'cuda'),
provider='CPUExecutionProvider',
).to(device)
pipe = ort_pipeline(
"automatic-speech-recognition",
model=model,
tokenizer=processor.tokenizer,
feature_extractor=processor.feature_extractor,
device=(-1 if device == 'cpu' else 0),
)
# Load audio file and run pipeline
audio = whisper.load_audio('tests/jfk.flac')
audio = whisper.pad_or_trim(audio)
outputs = pipe([audio])
print(outputs)
```
Note: In order to use these changes with Optimum, it is recommended to
use Optimum from source to have the following changes:
- https://github.com/huggingface/optimum/pull/872
- https://github.com/huggingface/optimum/pull/920
### Motivation and Context
This PR helps the following issues:
- https://github.com/microsoft/onnxruntime/issues/15100
- https://github.com/microsoft/onnxruntime/issues/15235
- https://github.com/huggingface/optimum/issues/869 (work in progress)
This PR can be used with the other currently merged Whisper PRs:
- https://github.com/microsoft/onnxruntime/pull/15247
- https://github.com/microsoft/onnxruntime/pull/15339
- https://github.com/microsoft/onnxruntime/pull/15362
- https://github.com/microsoft/onnxruntime/pull/15365
- https://github.com/microsoft/onnxruntime/pull/15427
This PR uses changes from the following merged PRs:
- https://github.com/microsoft/onnxruntime/pull/14198
- https://github.com/microsoft/onnxruntime/pull/14146
- https://github.com/microsoft/onnxruntime/pull/14201
- https://github.com/microsoft/onnxruntime/pull/14928 (this introduced
the new multi-head attention spec)
2023-04-19 00:13:54 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / t r a n s f o r m e r s / t e s t _ d a t a / m o d e l s / w h i s p e r
2021-08-06 15:30:27 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / e a g e r _ t e s t
2023-11-19 07:39:04 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / t r a n s f o r m e r s / t e s t _ d a t a / m o d e l s / c o n f o r m e r
2018-11-20 00:48:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { O N N X R U N T I M E _ R O O T } / _ _ i n i t _ _ . p y
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e /
2024-06-27 20:50:53 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { R E P O _ R O O T } / r e q u i r e m e n t s . t x t
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } >
2018-11-20 00:48:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { R E P O _ R O O T } / T h i r d P a r t y N o t i c e s . t x t
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e /
2019-10-20 14:58:36 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { R E P O _ R O O T } / d o c s / P r i v a c y . m d
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e /
2018-11-20 00:48:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { R E P O _ R O O T } / L I C E N S E
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e /
2018-11-20 00:48:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ b a c k e n d _ s r c s }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / b a c k e n d /
2018-11-20 00:48:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ s r c s }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
2021-09-02 16:54:32 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y _ i f _ d i f f e r e n t
$ { C M A K E _ B I N A R Y _ D I R } / o n n x r u n t i m e / c a p i / _ p y b i n d _ s t a t e . p y
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
2018-11-20 00:48:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e >
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
2018-11-20 00:48:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ d a t a s e t s _ s r c s }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / d a t a s e t s /
2018-11-20 00:48:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ d a t a s e t s _ d a t a }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / d a t a s e t s /
2018-11-20 00:48:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t o o l s _ s r c s }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s /
2021-04-30 04:23:54 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
2022-02-17 21:35:25 +00:00
$ { o n n x r u n t i m e _ m o b i l e _ u t i l _ s r c s }
2021-04-30 04:23:54 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s /
2022-03-15 05:52:12 +00:00
# append the /tools/python/utils imports to the __init__.py that came from /onnxruntime/tools.
# we're aggregating scripts from two different locations, and only include selected functionality from
# /tools/python/util. due to that we take the full __init__.py from /onnxruntime/tools and append
# the required content from /tools/python/util/__init__append.py.
C O M M A N D $ { C M A K E _ C O M M A N D } - E c a t
$ { R E P O _ R O O T } / t o o l s / p y t h o n / u t i l / _ _ i n i t _ _ a p p e n d . p y > >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s / _ _ i n i t _ _ . p y
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ q d q _ h e l p e r _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s / q d q _ h e l p e r s /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
2022-02-17 21:35:25 +00:00
$ { o n n x r u n t i m e _ m o b i l e _ h e l p e r s _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s / m o b i l e _ h e l p e r s /
2021-04-30 04:23:54 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ o r t _ f o r m a t _ m o d e l _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s / o r t _ f o r m a t _ m o d e l /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y _ d i r e c t o r y
$ { O N N X R U N T I M E _ R O O T } / c o r e / f l a t b u f f e r s / o r t _ f l a t b u f f e r s _ p y
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s / o r t _ f o r m a t _ m o d e l / o r t _ f l a t b u f f e r s _ p y
2020-07-09 04:42:53 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ q u a n t i z a t i o n _ s r c }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n /
2020-09-01 16:07:46 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ q u a n t i z a t i o n _ o p e r a t o r s _ s r c }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n / o p e r a t o r s /
2021-03-19 08:09:11 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ q u a n t i z a t i o n _ c a l _ t a b l e _ f l a t b u f f e r s _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n / C a l T a b l e F l a t B u f f e r s /
2023-12-12 16:43:04 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ q u a n t i z a t i o n _ f u s i o n s _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n / f u s i o n s /
[Quantization] Tensor quant overrides and QNN EP quantization configuration (#18465)
### Description
#### 1. Adds `TensorQuantOverrides` extra option
Allows specifying a dictionary of tensor-level quantization overrides:
```
TensorQuantOverrides = dictionary :
Default is {}. Set tensor quantization overrides. The key is a tensor name and the value is a
list of dictionaries. For per-tensor quantization, the list contains a single dictionary. For
per-channel quantization, the list contains a dictionary for each channel in the tensor.
Each dictionary contains optional overrides with the following keys and values.
'quant_type' = QuantType : The tensor's quantization data type.
'scale' = Float : The scale value to use. Must also specify `zero_point` if set.
'zero_point' = Int : The zero-point value to use. Must also specify `scale` is set.
'symmetric' = Bool : If the tensor should use symmetric quantization. Invalid if also
set `scale` or `zero_point`.
'reduce_range' = Bool : If the quantization range should be reduced. Invalid if also
set `scale` or `zero_point`.
'rmax' = Float : Override the maximum real tensor value in calibration data.
Invalid if also set `scale` or `zero_point`.
'rmin' = Float : Override the minimum real tensor value in calibration data.
Invalid if also set `scale` or `zero_point`.
```
- All of the options are optional.
- Some combinations are invalid.
- Ex: `rmax` and `rmin` are unnecessary if the `zero_point` and `scale`
are also specified.
Example for per-tensor quantization overrides:
```Python3
extra_options = {
"TensorQuantOverrides": {
"SIG_OUT": [{"scale": 1.0, "zero_point": 127}],
"WGT": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
"BIAS": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
},
}
```
Example for per-channel quantization overrides (Conv weight and bias):
```Python3
extra_options = {
"TensorQuantOverrides": {
"WGT": [
{
"quant_type": quantization.QuantType.QUInt8,
"rmin": 0.0,
"rmax": 2.5,
"reduce_range": True,
},
{
"quant_type": quantization.QuantType.QUInt8,
"rmin": 0.2,
"rmax": 2.55,
"reduce_range": False,
},
],
"BIAS": [
{"zero_point": 0, "scale": 0.000621},
{"zero_point": 0, "scale": 0.23},
],
},
}
```
#### 2. Adds utilities to get the default QDQ configs for QNN EP
Added a `quantization.execution_providers.qnn.get_qnn_qdq_config` method
that inspects the model and returns suitable quantization
configurations.
Example usage:
```python3
from quantization import quantize, QuantType
from quantization.execution_providers.qnn import get_qnn_qdq_config
qnn_config = get_qnn_qdq_config(input_model_path,
data_reader,
activation_type=QuantType.QUInt16,
weight_type=QuantType.QUInt8)
quantize(input_model_path,
output_model_path,
qnn_config)
```
### Motivation and Context
Make it possible to create more QDQ models that run on QNN EP.
---------
Signed-off-by: adrianlizarraga <adlizarraga@microsoft.com>
2023-12-05 01:54:58 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ q u a n t i z a t i o n _ e p _ q n n _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / q u a n t i z a t i o n / e x e c u t i o n _ p r o v i d e r s / q n n /
2020-09-10 22:42:15 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ s r c }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s /
2023-02-07 15:49:15 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ b a r t _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / b a r t /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ b e r t _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / b e r t /
2022-04-20 18:09:26 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ g p t 2 _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / g p t 2 /
2023-08-23 01:05:11 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ l l a m a _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / l l a m a /
Sync ORTModule branch with master and fix tests (#6526)
* Deprecate Python global configuration functions [Part 1] (#5923)
Enable options to be set via execution provider (EP)-specific options and log deprecation warning from current global configuration functions.
* remove dnnl_dll_path from post build copy (#6142)
* Model Fusion For Bart (#6105)
Fusion fix for Bart models
* Unify IExecutionProvider and IExecutionProviderFactory interfaces (#6108)
* Remove Provider_IExecutionProvider and make the internal IExecutionProvider usable by shared providers
* Change Provider_IExecutionProviderFactory to be the core version.
* Enable running the mnist_training sample without cuda (#6085)
Signed-off-by: George Nash <george.nash@intel.com>
* nnapi add min max support (#6117)
* Fix CUDA test hang: (#6138)
- Make condition check in `CUDAAllocatorTest` to ensure CUDA device is present.
* Fix TensorRT kernel conflict issue for subgraphs of control flow operators (#6115)
* add static subgraph kernel index
* change kernel naming to avoid conflicts
* Add gradient registration for Abs. (#6139)
* Partition initial optimizer state for Zero-1 (#6093)
* Initial changes
* Working changes
* Working changes
* Cleanup
* fix windows CI
* Review comments
* review comments
* Fix edge case in BFCArena where allocation failures could lead to an infinite loop. (#6145)
#4656
* Revert "work around of the build break in mac (#6069)" (#6150)
This reverts commit 3cae28699bed5de1fcaadb219fa69bae0fc3cee8.
* Fix clean_docker_image_cache.py detection of image pushes. (#6151)
Fix clean_docker_image_cache.py detection of image pushes. They were being ignored because the expected HTTP status code was wrong. For pushes, it's 201 instead of 200.
* MLAS: add NEON version of int8 depthwise convolution (#6152)
* Using a map of of ops to stages as input of partition function. (#5940)
* New partition algorithm running before AD
* Convert cut_group_info into device map. Work in progress -- works for bert-tiny with pp=2
* Removing code for partition of bwd graphs
* Remove old code
* Adding some verification code
* Handle Shared Initializer
* Renaming rank with stage
* Added first unit test
* new test
* redundant check
* undo change in bert
* Moved cut-based partition to testing utils file
Co-authored-by: xzhu1900
Co-authored-by: wschin
* New conversion function and tests
* minor
* remove test that is not needed2
* improve GetDeviceAssignment and PR comments
* minor changes
* PR comments
* improving documentation and variable naming
* add documentation
* Variable naming and docs
* more doc improvements
* more doc improvements
* missing static cast
* Fix test file for windows
* Fix test file for windows
* Fix test file for windows
* stage id is not the same as rank id
* PR comments
* PR comments
* More comments
* More comments
* Minor fix to satisfy c++14 (#6162)
* Deprecating Horovod and refactored Adasum computations (#5468)
deprecated horovod submodule
refactored adasum logic to be ort-native
added tests for native kernel and e2e tests
* Update TensorRT-ExecutionProvider.md (#6161)
* Bugfix for topk cuda kernel (#6164)
* fix the issue that std::numeric_limits cannot handle half type
* adding a test
Co-authored-by: Du Li <duli@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Revert "Fuse MatMulIntegerToFloat only when scales are scalar (#6008)" (#6169)
This reverts commit f2dcba7afe0d42ebdaaef0c6cdf913a1156c9e98.
* Remove ignored build warnings for pybind on Mac (#6165)
* save_checkpoint, load_checkpoint and aggregate_checkpoints (#6136)
* save_checkpoint and load_checkpoint implementations
* checkpoint aggregation logic
* unit tests for save_checkpoint, load_checkpoint and aggregate_checkpoints
* Don't try to bind unused inputs in the Training frontend (#6166)
* Update documentation for contributing a PR and add deprecation notices for PyOp and ORT server. (#6172)
* aggregate model states only for the case when mixed precision was true (#6176)
* [NNAPI EP] Enable per-channel quantization for QlinearConv (#6155)
* Enable qlinearconv per-channel quantization
* Fix the android CI test failure
* Add Android Version Check for Per-Channel Quant
* Address PR comments
* Fix some minor issues
* Add verification of per-channel zero points
* Make the error tolerance configurable
* Fix typo in BERT pretraining script (#6175)
A misplaced `}` meant that the `'enable_adasum'` option was interpreted incorrectly, causing the test to fail.
* Update get_docker_image.py to enable use without image cache container registry. (#6177)
Update get_docker_image.py to enable use without image cache container registry.
* Helper for compiling EP to generate deterministic unique ids for use in MetaDef names (#6156)
* Create a helper for generating unique ids that can be used by an EP that creates compiled nodes and needs ids to be deterministic for a model when used in multiple sessions.
Added to IExecutionProvider as this can potentially be used by all compiling EPs and is more robust than a simplistic counter (although EP implementer is free to choose either approach).
* Restructure the helper so it can be called across the EP bridge.
Add ability to call id generation helper from EP bridge
- convert DNNL EP to use helper to validate
Address issue where a new Model may be loaded into the same address as a previous one.
- hash the bytes in the Graph instance (1728 bytes currently) to use as the key to the full hash for the model
Add lock around id generation to ensure no issues if multiple sessions partitions graphs at exactly the same time.
- Extremely unlikely but would be hard to debug and the locking cost is not an issue as it's only incurred during graph partitioning and not execution.
* Backend APIs for checkpointing (#5803)
* Add backend API GetOptimizerState and GetModelState
* add GetPartitionInfoMap
* Android coverage dashboard (#6163)
* Write the report to a file.
* Post code coverage to the Dashboard database.
* Add usage details of unified MCR container image (#6182)
Going forward, a single unifed docker image will be published in
MCR. The hardware accelerator target choice will have to be made
in the application using OpenVINO EP's runtime config options.
* improve perf for softmax (#6128)
* improve perf for both gathergrad and softmax
* revert the change in gathergrad and will be done in another PR.
* address comments from code review.
* Tune fast Gelu to use exp(x) instead of tanh(x) on Rocm platform (#6174)
* tune fast gelu to use exp(x) instead of tanh(x) on rocm
* update to use expression 2/(1+exp(-2x))-1 for stability
* Add Status.csv to EP Perf Tool (#6167)
* merge master, keep postprocess status commit
* download float16.py everytime
* removing hardcoded values
* Lochi/quantization tool for trt (#6103)
* Initial implementation of generating calibration dynamic range table
* Initialize validation support for Quantization
* Initialize validation support for Quantization (cont.)
* Improve validation support for Quantization
* Improve validation support for Quantization
* Rewrite/Refine for calibration and validation
* Rewrite/Refine for calibration and validation (cont.)
* Refine code
* Refine code
* Add data reader for BERT
* Add flatbuffers to serialize calibration table
* Refine code and add BERT evaluation
* Refine the code
* minor modification
* Add preprocess/postprocess of vision team yolov3 and refine the code
* Update annotation
* Make bbox cooridates more accurate
* Fix bug
* Add support of batch processing
* Batch processing for model zoo yolov3
* Add batch inference for evaluation
* Refine the code
* Add README
* Add comments
* Refine the code for PR
* Remove batch support checking in data_reader and refine the code
* Refine the code for PR
* Refine the code for PR review
Co-authored-by: Olivia Jain <oljain@microsoft.com>
* Implement ScatterND for CUDA EP (#6184)
* Condition fix in Resize operator (#6193)
* Clean up checkpoint tests to use the new checkpoint functions (#6188)
* add deprecation warning for old checkpoint functions
* update all the distributed checkpoint tests to use new checkpoint functions
* Implement comparing outputs that are sequence of maps of strings to floats (#6180)
* Implement conversion from ortvalue to Itensor for string tensors and comparing sequence of maps of strings to floats
* PR comments
* Dockerfile to build onnxruntime with ROCm 4.0
* Add ability to skip GPU tests based on GPU adapter name (#6198)
* Implement conversion from ortvalue to Itensor for string tensors and comparing sequence of maps of strings to floats
* PR comments
* Add ability to skip gpu tests according to adapter description
* spacing
* spacing
* spacing
* Openvino ep 2021.2 (#6196)
* Enabling fasterrcnn variant and vehicle detector
* changes for 2021_2 branch
* yolov3_pytorch commit
* fixed braces in basic_backend.cc
* ci information added
* faster rcnn variant and vehicle detector changes were made in 2021.1 and not in 2021.2
* some changes to support unit tests
* disable some tests which are failing
* fix myriad tests for vehicle detector
* Did some cleanup
*cleaned up comments
*Disabled Add_Broadcast_0x1 and Add_Broadcast_1x0
tests on MYRIAD_FP16 backend due to a bug
*cleaned up capability_2021_2.cc file
*Removed extra conditions which were added
for some validation in backend_utils
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* yolov3 pytorch workaround to ensure that the output names are matched
* gemmoptest fixed on myriad
* Fixed MYRIADX CPP Test Failures
*Expand,GatherND,Range,Round op's
are only supported in model
*where op with float input data
types are not supported and fixed
*Scatter and ScatterElements op's with
negative axis are fixed
*Reshape op with 0 dim value are not
supported and fixed
*Disabled InstanceNorm_2 test on MYRIADX
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* make changes to yolov3 pytorch
* Fixed python unit tests
*Fixed failing python tests on vpu,
GPU and CPU
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Fixes POW op failures on GPU_FP16
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Clean up capability_2021_2.cc
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Updated docx for MultiThreading option
*Added extra info on setting the num_of_threads
option using the API and it's actual usage
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* fixed slice and removed extra prints
* Disabled failing python tests
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Minor changes added in capabilty_2021_2
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* made changes to slice to avoid failures
* Disabling FP16 support for GPU_FP32
->Inferencing an FP16 model on GPU_FP32
leads to accuracy mismatches. so, we would
rather use GPU_FP16 to infer an FP16 model
on GPU Device
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Updated docx for Inferencing a FP16 Model
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* fix for mask rcnn
* Script for installing openvino from source
* Updated with openvino 2021.2 online installation
* code comment fixes
fixed accuracy mismatch for div
* Update OpenvinoEP-ExecutionProvider.md
updated for 2021.2 branch
* Update README.md
updated dockerfile documentation
* Update BUILD.md
build.md update documentation
* permissiong change of install_openvino.sh
* made changes to align with microsoft onnxruntime changes
* Updated with ov 2021.2.200
Co-authored-by: suryasidd <surya.siddharth.pemmaraju@intel.com>
Co-authored-by: sfatimar <sahar.fatima@intel/com>
Co-authored-by: MaajidKhan <n.maajidkhan@gmail.com>
Co-authored-by: mohdansx <mohdx.ansari@intel.com>
* Fix a memory leak in test_inference.cc (#6201)
* Fix a memory leak in test_inference.cc
* Use TArray in AMD element-wise kernels, rather than manually copying memory to device.
* Remove most ROCm-specific element-wise code and reuse CUDA element-wise code.
* Minor change to improve performance for operator Pad. (#5537)
* small improvment for pad
* Support double for operators Log, Reciprocal, Sum (CPU) (#6032)
* Support double for operators Log, Reciprocal, Sum
* remove tesdt erf_double
* Support double for operators Where, LpNormalisation (#6034)
* Support double for operators Relu, Tanh, Sigmoid (#6221)
* Fix ImportError in build.py (#6231)
There is a possible ImportError where build.py can import the wrong 'util' package if there are others present in `sys.path` already
* Removed executor todo that looks dead. (#6234)
* Remove MKLML/openblas/jemalloc build config (#6212)
* Remove python 3.5
* Update the readme file
* Upgrade build.py to assert for python 3.6+
Upgrade build.py to assert for python 3.6+
as python 3.5 cannot build anymore todays master.
* Support MLFloat16 type in Pow opset-12 CUDA kernel (#6233)
* MLAS: handle MlasGemm(M/N/K==0) cases (#6238)
* Support double for operator TopK + fix one bug in TopK implementation for GPU for double (#6220)
* Support double for operator TopK
* add static classes for topk/double
* fix cast issue in topk
* Support double for operator Gemm + fix bug in gemm implementation for cuda, rocm when sizeof(type) != sizeof(float) (#6223)
* Support double for operator Gemm
* fix type size while copying data in gemm operator for GPU
* fix type in gemm implementation for rocm
* Support double for operator ReduceMean, ReduceLogSumExp (#6217)
* Support double for operators ReduceMean, ReduceLogSumExp
* Support double for operator ArgMin (#6222)
* Support double for operator ArgMin
* add test specifically for double
* add new test on pai-excluded-tests.txt
* Update BUILD.md
* Update manylinux docker image to the latest (#6242)
* Fix allocator issue for TensorRT IOBinding (#6240)
* Fix issue: https://github.com/microsoft/onnxruntime/issues/6094
Root cause: we didn't expose the OrtMemoryInfo for TRT, so it will cause issue if user want use IObinding for Tensorrt.
Short term fix, add the OrtMemoryInfo for TRT. Long term should unify the allocator for CUDA and TRT
* Tune BiasGeluGradDx kernel in approximation mode to avoid tanh(...) on Rocm (#6239)
* bias gelu grad use exp(...) instead
* update cuda to rocm
* missing semicolon
* comment
* remove dockerfile
* missing factor of two
* Refactor EP Perf Tool (#6202)
* merge master, keep postprocess status commit
* download float16.py everytime
* using variables to reference eps
* adding ACL EP to ep perf tool
* accuracy with absolute tolerance configurable
* add acl to dict + remove commented line
* Documentation for distributed CI tests pipeline (#6140)
* Remove a debug log in provider_test_utils.cc (#6200)
* Add the Concat Slice Elimination transform, fix constant_folding transform (#5457)
* Add concat slice transform + test
* Cosmetic improvements in concat slice transform
* Remove unrelated file, fix comment, fix constant folding bug
* Add test onnx graph
* fix windows build
* Review comments
* review comment
Co-authored-by: Aishwarya <aibhanda@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add MakeStringLite which uses current locale, update some MakeString call sites to use it instead. (#6252)
* Add MakeStringLite which uses current locale, update macros to use that to generate messages.
* Convert calls to MakeStringLite().
* Liqun/speech model loop to scan (#6070)
Provide a tool to convert Loop to Scan for Nuphar performance
Fix Nuphar CI pipeline failures.
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* model parallel refinement (#6244)
* Megatron Transformation as a seperate step
* remove useless header
* clang formating
* Re-Structure megatron transformer for subsquent changes
* fix comments
* Allow querying a GraphProto's doc_string as part of ModelMetadata (#6248)
* Fix Linux/Mac error message on input type mismatch (#6256)
* add bfloat16 to gathergrad type constrains (#6267)
Co-authored-by: Cheng Tang <chenta@microsoft.com>
* Fix VS 2017 build break (#6276)
* Deprecate Python global configuration functions [Part 2] (#6171)
Update Python API to allow more flexibility for setting providers and provider options.
The providers argument (InferenceSession/TrainingSession constructors, InferenceSession.set_providers()) now also accepts a tuple of (name, options dict).
Fix get_available_providers() API (and the corresponding function in the C API) to return the providers in default priority order. Now it can be used as a starting point for the providers argument and maintain the default priority order.
Convert some usages of the deprecated global configuration functions to use EP-specific options instead.
Update some EP-specific option parsing to fail on unknown options.
Other clean up.
* Add script to preprocess python documentation before publishing (#6129)
* add script to preprocessing python documentation before publishing
* rename past to past_key_values for GPT-2 (#6269)
rename past to past_key_values for transformers 4.*
* Rename MakeString and ParseString functions. (#6272)
Rename MakeString to MakeStringWithClassicLocale, MakeStringLite to MakeString, *ParseString to *ParseStringWithClassicLocale.
Add missing pass-through versions of MakeStringWithClassicLocale for string types.
* Increase timeout for Linux GPU CUDA11 build. (#6280)
* Add helper to compare model with different precision (#6270)
* add parity_check_helper.py
* add real example
* remove lines
* Fix Min/Max CPU kernels for float16 type (#6205)
* fix data_ptr assertion error for past_sequence_length=0 in GPT-2 (#6284)
fix io binding crash for past_sequence_length=0
* A list of changes in transformers tool (#6224)
* longformer fp16 e2e
* add fp16/fp32 parity check helper file
* excludes nodes with subgraph in profiling
* use onnxconverter_common to do fp32->fp16
* add version check for onnxconverter_common
* remove helper file
* add pkg installation on notebooks and script
* Workaround for static_cast<double>(half)
* Add workaround to remove ROCm-specific binary-elementwise files.
* Update nuget build (#6297)
1. Update the ProtoSrc path. The old one is not used anymore.
2. Regenerate OnnxMl.cs
3. Delete some unused code in tools/ci_build/build.py
4. Avoid set intra_op_param.thread_pool_size in ModelTests in OpenMP build.
5. Fix a typo in the C API pipeline.
* Enable ONNX backend test of SequenceProto input/output (#6043)
* assert sequence tensor and remove skips
* update testdata json
* use ONNX 1.8 in cgmanifest.json
* use previous commit to workaround
* update ONNX commit ID in docker
* skip test_maxpool_2d_dilations test for now
* update function name
* add --sequence_lengths option (#6285)
* more dtype for Equal CUDA kernel (#6288)
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
* Force reinstall onnx python package on Windows (#6309)
* update transformers required package versions (#6315)
* Remove abs in LpPool (#6303)
* Support 1D input for Conv + Mul/Add fusion optimizer with test (#6295)
* Support 1D input (N C H) for Conv + Mul/Add fusion optimizer with test cases and test models.
* Add longformer to python package (#6314)
* add longformer to python package
* move test related script and data to a new folder
* Avoid false sharing on thread pool data structures (#6298)
Description: This change adds alignment and padding to avoid false sharing on fields in the thread pool. It also adds a new microbenchmark to profile thread-pool performance over short loops.
Motivation and Context
MobileNet on a 2*12-core system showed a performance gap between the ORT thread pool and OpenMP. One cause appeared to be false sharing on fields in the thread pool: ThreadPoolParallelSection::tasks_finished (which the main thread spins on waiting for workers to complete a loop), and the RunQueue::front_ and back_ fields (used respectively by the worker thread and the main thread).
The additional micro-benchmark BM_ThreadPoolSimpleParallelFor tests performance of loops of different sizes at different thread counts. The results below are on a machine with 2*14-core processors (E5-2690 v4) running with 1, 14, 15, and 28 threads. For each test, the microbenchmark has N threads run a loop with N iterations; hence a perfect result is for the time taken to be constant as additional threads are added (although we will also see power management effects helping at very low thread counts). The loop durations (100000, 10000, 1000) correspond roughly to 200us, 20us, and 2us on this machine.
Before change:
BM_ThreadPoolSimpleParallelFor/1/1/100000/real_time 17153 us 17154 us 32
BM_ThreadPoolSimpleParallelFor/14/14/100000/real_time 22553 us 22553 us 30
BM_ThreadPoolSimpleParallelFor/15/15/100000/real_time 21521 us 21521 us 29
BM_ThreadPoolSimpleParallelFor/28/28/100000/real_time 24111 us 24111 us 24
BM_ThreadPoolSimpleParallelFor/1/1/10000/real_time 1719 us 1719 us 407
BM_ThreadPoolSimpleParallelFor/14/14/10000/real_time 3409 us 3409 us 200
BM_ThreadPoolSimpleParallelFor/15/15/10000/real_time 3541 us 3541 us 201
BM_ThreadPoolSimpleParallelFor/28/28/10000/real_time 4576 us 4576 us 151
BM_ThreadPoolSimpleParallelFor/1/1/1000/real_time 174 us 174 us 4017
BM_ThreadPoolSimpleParallelFor/14/14/1000/real_time 1586 us 1586 us 402
BM_ThreadPoolSimpleParallelFor/15/15/1000/real_time 1586 us 1586 us 397
BM_ThreadPoolSimpleParallelFor/28/28/1000/real_time 2864 us 2864 us 232
After change:
BM_ThreadPoolSimpleParallelFor/1/1/100000/real_time 17160 us 17160 us 33
BM_ThreadPoolSimpleParallelFor/14/14/100000/real_time 20989 us 20989 us 31
BM_ThreadPoolSimpleParallelFor/15/15/100000/real_time 22286 us 22286 us 31
BM_ThreadPoolSimpleParallelFor/28/28/100000/real_time 24631 us 24631 us 25
BM_ThreadPoolSimpleParallelFor/1/1/10000/real_time 1718 us 1718 us 407
BM_ThreadPoolSimpleParallelFor/14/14/10000/real_time 2868 us 2868 us 242
BM_ThreadPoolSimpleParallelFor/15/15/10000/real_time 2907 us 2907 us 240
BM_ThreadPoolSimpleParallelFor/28/28/10000/real_time 3872 us 3872 us 186
BM_ThreadPoolSimpleParallelFor/1/1/1000/real_time 175 us 175 us 3938
BM_ThreadPoolSimpleParallelFor/14/14/1000/real_time 933 us 933 us 659
BM_ThreadPoolSimpleParallelFor/15/15/1000/real_time 912 us 912 us 591
BM_ThreadPoolSimpleParallelFor/28/28/1000/real_time 1976 us 1976 us 317
* fix opset imports for function body (#6287)
* fix function opsets
* add tests and update onnx
* changes per review comments
* add comments
* plus updates
* build fix
* Remove false positive prefast warning from threadpool (#6324)
* Java: add Semmle to Java publishing pipelines (#6326)
Add Semmle to Java API pipeline
Add security results publishing and add Java GPU.
* Quantization support for split operator with its NHWC support (#6107)
* Make split working for quantization.
* NHWC transformer support for split operator
* Refactor some according to Feedback. Will add test cases soon.
* Fix build error on windows.
* Add test case for split op on uint8_t support
* Add nhwc_transformer_test for split uint8_t support
* Some change according to PR feedbacks.
* Liqun/enable pipeline parallel test (#6331)
enable pipeline parallel test
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Use onnxruntime_USE_FULL_PROTOBUF=OFF for the cuda execution provider (#6340)
This removes a special case of the cuda EP.
* MLAS: add fallback implementation for quantized GEMM (#6335)
Add a non-vectorized version of the kernel used for the quantized version of MlasGemm.
* Delete float16.py (#6336)
No longer needed. Also doesn't pass policheck.
* Enable add + softmax fusion for Rocm platform (#6259)
* add bias softmax; tests appear to pass
* check fusion occurs for rocm as well
* check for rocm provider compatible as well
* build for cpu scenario as well
* try again; broader cope
* proper scope on kGpuExecutionProvider
* been editing wrong file
* remove commented #include lines
* try again due to mac os ci error
* try again
* test fusion both cuda and rocm to avoid mac ci error
* add external data support to tensor proto utils (#6257)
* update unpack tensor utilities to support loading external data
* more updates
* fix test
* fix nuphar build
* minor build fix
* add tests
* fix Android CI
* fix warning
* fix DML build failure and some warnings
* more updates
* more updates
* plus few updates
* plus some refactoring
* changes per review
* plus some change
* remove temp code
* plus updates to safeint usage
* build fix
* fix for safeint
* changed wording. (#6337)
* Remove OpSchema dummy definition. Only needed for Function now, and we can just exclude the method in Function (#6321)
* remove gemmlowp submodule (#6341)
* [NNAPI] Add pow support (#6310)
* Add support for running Android emulator from build.py on Windows. (#6317)
* fix the pipeline failure (#6346)
* Train BERT Using BFloat16 on A100 (#6090)
* traing bert using bf16
* Adam support bf16
* bugfix
* add fusedmatmul support
* fix after merge from master.
* bugfix
* bugfix after merge from master
* fast reduction for bf16.
* resolve comments
* fix win build
* bugfix
* change header file.
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
* Fix DerefNullPtr issues raised by SDLNativeRules. (#6348)
* update quantize to support basic optimization and e2e example for image classification (#6313)
update the resnet50-v1 to standard one from onnx zoo.
add an example for mobilenet
run basic optimization before quantization
fix a bug in Clip
* Enable graph save for orttrainer (#6333)
* Enable graph save for orttrainer
* Fix CI
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add PREfast to python packaging pipeline (#6343)
* Add PREfast to python packaging pipeline
* fix longformer benchmark io_binding output_buffers (#6345)
* fix longformer benchmark io_binding output_buffers
* format
* import benchmark_helper from parent directory.
* Use readelf for minimal build binary size checks. (#6338)
* Use readelf for minimal build binary size checks.
The on-disk size grows in 4KB chunks which makes it hard to see how much growth an individual checkin causes.
Only downside is that the sum of the sections is larger than the on-disk size (assumably things get packed smaller on disk and some of the section alignment constraints can be ignored)
* Remove unused function
* Java: Set C language warnings to W4 and adjust JNI code (#6347)
Set /W3 for C language and fix up JNI warnings.
* Pipeline Parallel Experimental Python API (#5815)
* Add create session to WinML telemetry to track WinML Usage (#6356)
* Fix one more SDL warning (#6359)
* fix -Wdangling-gsl (#6357)
* Add python example of TensorRT INT8 inference on ResNet model (#6255)
* add trt int8 example on resnet model
* Update e2e_tensorrt_resnet_example.py
* remove keras dependency and update class names
* move ImageNetDataReader and ImageClassificationEvaluator to tensorrt resnet example
* simplify e2e_tensorrt_resnet_example.py
* Update preprocessing.py
* merge tensorrt_calibrate
* Update calibrate.py
* Update calibrate.py
* generalize calibrate
* Update calibrate.py
* fix issues
* fix formating
* remove augment_all
* This added telemetry isn't needed (#6363)
* Wezuo/memory analysis (#5658)
* merged alloc_plan
* pass compilation
* Start running, incorrect allocation memory info
* add in comments
* fix a bug of recording pattern too early.
* debugging lifetime
* fix lifetime
* passed mnist
* in process of visualization
* Add code to generate chrome trace for allocations.
* in process of collecting fragmentation
* before rebuild
* passed mnist
* passed bert tiny
* fix the inplace reuse
* fix the exception of weight in pinned memory
* add guards to ensure the tensor is in AllocPlan
* add customized profiling
* debugging
* debugging
* fix the reuse of differnt location type
* add rank
* add the rank
* add fragmentation
* add time_step_trace
* Add summary for each execution step (total bytes, used/free bytes).
* add top k
* change type of top k parameter
* remove prints
* change heap to set{
* add the name pattern
* add the useage for pattern
* add partition
* change to static class
* add custom group
* remove const
* update memory_info
* in process of adding it as runtime config
* change the memory profiling to be an argument
* add some comments
* add checks to recored meomry_info in traaining session
* set the "local rank setting" to correct argument.
* addressing comments
* format adjustment
* formatting
* remove alloc_interval
* update memory_info.cc to skip session when there is no tensor for a particular memory type
* fix memory_info multiple iteration seg-fault
* consolidate mainz changes
* fixed some minor errors
* guard by ORT_MINIMAL_BUILD
* add ORT_MEMORY_PROFILE flag
* added compiler flag to turn on/off memory profiling related code
* clean up the code regarding comments
* add comments
* revoke the onnx version
* clean up the code to match master
* clean up the code to match master
* clean up the code to match master
Co-authored-by: Jesse Benson <benson.jesse@gmail.com>
Co-authored-by: Wei Zuo <wezuo@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-mgtbby.eastus.cloudapp.azure.com>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-yclzsf.eastus.cloudapp.azure.com>
* Support MLFloat16 in CumSum Cuda op for Opset 14 (#6355)
* Add CumSum-14 for Cuda
* fix convert_common version retrival (#6382)
* Refine auto_pad based pad computation in ConvTranspose (#6305)
* Fix SDL warning (#6390)
* Add max_norm for gradient clipping. (#6289)
* add max_norm as user option for gradient clipping
* add adam and lamb test cases for clip norm
* add frontend tests
* Add the custom op project information (#6334)
* Dont use default string marshalling in C# (#6219)
* Fix Windows x86 compiler warnings in the optimizers project (#6377)
* [Perf] Optimize Tile CPU and CUDA kernels for a corner case (#6376)
* Unblock Android CI code coverage failure (#6393)
* fix build on cuda11 (#6394)
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
* Load the model path correctly (#6369)
* Fix some compile warnings (#6316)
* OpenVino docker file changes to bypass privileged mode
Description: Builds and installs libusb without UDEV support, which is used for communicating with the VPU device.
Motivation and Context
This enables the resulting docker container to be run without '--privileged' and '--network host' options which may not be suitable in deployment environments.
* Megatron checkpointing (#6293)
* Add bart fairseq run script
* Add frontend change to enable megatron
* Initial changes for checkpointing
* Megatron optim state loading, checkpoint aggregation, frontend distributed tests for H, D+H
* Add load_checkpoint changes
* Fix CI
* Cleanup
* Fix CI
* review comments
* review comments
* review comments:
* Fix generate_submodule_cgmanifest.py Windows issues. (#6404)
* Continue memory planning when unknown shape tensor is encountered. (#6413)
* Reintroduce experimental api changes and fix remote build break (#6385)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* Add support for custom ops to minimal build. (#6228)
* Add support for custom ops to minimal build.
Cost is only ~8KB so including in base minimal build.
* enable pipeline to run quantization tests (#6416)
* enable pipeline to run quantization tests
setup test pipeline for quantization
* Minor cmake change (#6431)
* Liqun/liqun/enable pipeline parallel test2 (#6399)
* enable data and pipeline parallism test
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Farewell TrainableDropout (#5793)
* Deprecate TrainableDropout kernel.
* Update bert_toy_postprocessed.onnx to opset 12.
* Add more dropout tests.
* Fix BiasDropout kernel.
Co-authored-by: Ubuntu <OrtTrainingDev3@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sherlock Huang <bahuang@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sergii Dymchenko <sedymche@microsoft.com>
* fix null dereference warning (#6437)
* Expose graph ModelPath to TensorRT shared library (#6353)
* Update graph_viewer.cc
* Update tensorrt_execution_provider.cc
* Update graph_viewer.h
* Update tensorrt_execution_provider.cc
* Update tensorrt_execution_provider.cc
* Update provider_api.h
* Update provider_bridge_ort.cc
* Update provider_interfaces.h
* Update provider_interfaces.h
* expose GraphViewer ModelPath API to TRT shared lib
* add modelpath to compile
* update
* add model_path to onnx tensorrt parser
* use GenerateMetaDefId to generate unique TRT kernel name
* use GenerateMetaDefId to generate unique TRT engine name
* fix issue
* Update tensorrt_execution_provider.cc
* remove GetVecHash
* Update tensorrt_execution_provider.h
* convert wchar_t to char for tensorrt parser
* update tensorrt parser to include latest changes
* fix issues
* Update tensorrt_execution_provider.cc
* merge trt parser latest change
* add PROVIDER_DISALLOW_ALL(Path)
* add tool for generating test data for longformer (#6415)
* only build experimental api in redist (#6465)
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
* Add an option to save the training graph after optimization (#6410)
* expose optimized_model_filepath in SessionOptions as `debug.graph_save_paths.model_with_training_graph_after_optimization_path` in `ORTTrainerOptions`
* Share allocator between CUDA EP & TRT EP. (#6332)
* Share allocator between CUDA EP & TRT EP.
limitation:
1. Does not cover the per-thread allocator created by CUDA EP, still need to figure out the way to remove it
2. Need to have more identifiers to make it able to share CPU allocator across all EPs
* fix max norm clipping test in python packaging pipeline test (#6468)
* fix python packaging pipeline
* make clip norm test compatabile with both V100 and M60 GPUs
* Initial version of CoreML EP (#6392)
* Bug 31463811: Servicing: Redist (Nuget) conflicts with Microsoft.AI.MachineLearning starting 21H1+ (#6460)
* update load library code to have the fullly qualified path
* make it work for syswow32
* git Revert "make it work for syswow32"
This reverts commit b9f594341b7cf07241b18d0c376af905edcabae3.
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
* dequantize 1st input of lstm back if it is quantized (#6444)
* [java] Adds support for OrtEnvironment thread pools (#6406)
* Updates for Gradle 7.
* Adding support for OrtThreadingOptions into the Java API.
* Fixing a typo in the JNI code.
* Adding a test for the environment's thread pool.
* Fix cuda test, add comment to failure.
* Updating build.gradle
* fix SDL native rule warning #6246 (#6461)
* fix SDL rule (#6464)
* use tickcount64 (#6447)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* Update pypi package metadata (#6354)
* Update setup file data
* add missing comma
* remove python 3.5
* fix typo bracket
* Delete nuget extra configs (#6477)
* Op kernel type reduction infrastructure. (#6466)
Add infrastructure to support type reduction in Op kernel implementations.
Update Cast and IsInf CPU kernels to use it.
* Fixing a leak in OnnxSequences with String keys or values. (#6473)
* Increase the distributes tests pipeline timeout to 120 minutes (#6479)
* [CoreML EP] Add CI for CoreML EP (macOS) and add coreml_flags for EP options (#6481)
* Add macos coreml CI and coreml_flags
* Move save debuggubg model to use environment var
* Move pipeline off from macos CI template
* Fix an issue building using unix make, add parallel to build script
* Fixed build break for shared_lib and cmpile warning
* Fix a compile warning
* test
* Revert the accidental push from another branch
This reverts commit 472029ba25d50f9508474c9eeceb3454cead7877.
* Add ability to track per operator types in reduced build config. (#6428)
* Add ability to generate configuration that includes required types for individual operators, to allow build size reduction based on that.
- Add python bindings for ORT format models
- Add script to update bindings and help info
- Add parsing of ORT format models
- Add ability to enable type reduction to config generation
- Update build.py to only allow operator/type reduction via config
- simpler to require config to be generated first
- can't mix a type aware (ORT format model only) and non-type aware config as that may result in insufficient types being enabled
- Add script to create reduced build config
- Update CIs
* merge e2e with distributed pipeline (#6443)
merge e2e with distributed pipeline
* Fix test breaks in Windows ingestion pipeline (#6476)
* fix various build breaks with Windows build
* fix runtime errors loading libraries from system32
* add build_inbox check to winml_test_common
* use raw string
* cleanup
* fix dll load
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
* Speed up the Mac CI runs (#6483)
* expose learningmodelpixelrange property (#5877)
* Fix of support api version bug for [de]quantize (#6492)
* SDL fixes: add proper casts/format specifiers (#6446)
* SDL annotation fixes (#6448)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* [OpenVINO-EP] Remove support for OpenVINO 2020.2 (#6493)
* Removed OpenVINO 2020.2 support
* Updated documentation and build.py
* Removed unnecessary libraries from setup.py
* Support pad operator in quantization and quantized nhwc transformer. Fix Pad operator bug. (#6325)
Support pad operator in quantization tool.
Support pad operator in quantized nhwc transformer.
Fix pad() operator bug when pad input's inner(right) most axis value is zero for Edge and Reflect mode, it copied wrong value to the cells to be padded. Note the Constant mode will not trigger this bug, as Edge/Reflect need copy value from the already copied array while Constant mode only fill specified value.
Add more test cases to cover pad() operator bug fixed here.
Fix quantization tools uint8/int8 value overflow issue when quantize weights in python.
* Improve work distribution for Expand operator, and sharded LoopCounter configuration (#6454)
Description: This PR makes two changes identified while looking at a PGAN model.
First, it uses ThreadPool::TryParallelFor for the main parallel loops in the Expand operator. This lets the thread pool decide on the granularity at which to distribute work (unlike TrySimpleParallelFor). Profiling showed high costs when running "simple" loops with 4M iterations each of which copied only 4 bytes.
Second, it updates the sharded loop counter in the thread pool so that the number of shards is capped by the number of threads. This helps make the performance of any other high-contention "simple" loops more robust at low thread counts by letting each thread work on its own "home" shard for longer.
Motivation and Context
Profiling showed a PGAN model taking 2x+ longer with the non-OpenMP build. The root cause was that the OpenMP build uses simple static scheduling of loop iterations, while the non-OpenMP build uses dynamic scheduling. The combination of large numbers of tiny iterations is less significant with static scheduling --- although still desirable to avoid, given that each iteration incurs a std::function invocation.
* Update document of transformer optimization (#6487)
* nuphar test to avoid test data download to improve passing rate (#6467)
nuphar test to avoid test data download to improve passing rate
* Fuse cuda conv with activation (#6351)
* optimize cuda conv by fused activation
* remove needless print out
* exclude test from cpu
* handle status error from cudnn 8.x
* add reference to base class
* add hipify
* [CoreML EP] Add support for some activations/Transpose, move some shared helpers from NNAPI to shared space (#6498)
* Init change
* Move some helper from nnapi ep to shared
* Add transpose support
* Fix trt ci build break
* Refine transformers profiler output (#6502)
* output nodes in the original order; grouped by node name
* add document for profiler
* Update to match new test setup. (#6496)
* Update to match new test setup.
* Add Gemm(7) manually for now.
Will fix properly on Monday. It's used by mnist.ort as that is created by optimizing mnist.onnx to level 1 causing 2 nodes to be replaced by a Gemm and the op to be missing from the required list as that is created using the original onnx model.
* Enable dense sequence optimized version of Pytorch exported BERT-L on AMD GPU (#6504)
* Permit dense seq optimization on BERT-L pytorch export by enabling ReduceSumTraining, Equal, and NonZero on AMD
* enable Equal tests
* enable fast_matrix_reduction test case
* Optimize GatherGrad for AMD GPU (#6381)
* optimize gathergrad
* address comments
Co-authored-by: Weixing Zhang <wezhan@microsoft.com>
* add explicit barriers for buffer overread and overrwrite (#6484)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* fix sdl bugs for uninitialized variables and returns (#6450)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* handle hr error conditions (#6449)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* Dnnl training (#6045)
* Add ReluGrad and ConvGrad ops for the dnnl provider
* the mnist sample is updated to add the --use_dnnl option that
will cause the sample to use the dnnl execution provider for
nodes that exist in dnnl provider.
* Added the ability to find forward ops. Dnnl backward gradient
ops require the forward primitive description and workspace
from the forward operation.
* Enable specifying the execution provider for Gradient Checker Tests
* Prevent memory leak when running dnnl_provider in training mode
Prevent creating a SubgraphPrimitivePool when the code is built with the
ENABLE_TRAINING build flag. Instead create a SubgraphPrimitive directly.
The SubgraphPrimitivePool was causing a pool of SubgraphPrimitives to be
stashed in a map for reuse. Due to the way the Training Loop uses threads
the pool of SubgraphPrimitives were not being reuse instead a new pool of
SubgraphPrimitives being created each run. The old pool was not instantly
freed. This behavior could be a language error when using thread_local
memory.
Signed-off-by: George Nash <george.nash@intel.com>
* Added fixes to maxpoolgrad and memory leak.
Maxpoolgrad will now pass all unit tests.
With the conv and convgrad disabled for dnnl, mnist is able to train till 95%
Signed-off-by: Chethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
* Fixed misc issues when testing training code with dnnl provider
* fix conv_grad dnnl tests with dilation to run dnnl execution provider
* update mnist training sample to accept convolution type models
convolution models require the input shape to be {1, 28, 28}
instead of the flat {728} image that is used for the gemm models
this will enable models that require the different shape by adding
`--model_type conv` to the command line when running the mnist sample.
(while testing a workaround was used see #4762)
* Disable weight caching in dnnl conv operator when using training
When training we can not use cached weights because the weight
will be updated each run. This re-enables dnnl Conv and ConvGrad Ops.
The weight caching was the source of the error from Conv when training.
* Fix issues found when building grad ops on Linux
* The dnnl_convgrad code was over using the scope operator
causing a compilation problem.
* The dnnl_maxpoolgrad code had a logic error that is was
comparing with the source description when it should have
been comparing with the destination despription.
* Update BUILD.md so it shows DNNL for training
* Updated the table of contents. Since the same providers
are listed twice. Once for Infrance and again for Training
an HTML anchor was added to distinguish the second header
from the first for the TOC.
* Fix build failure when not using --enable-training build option
* reorganize the gradient operators so they are grouped together
* Fix issues found when running onnx_backend_test_series.py
* Pooling code only supports 2 outputs when built with --enable-training
* Address code review feedback
* class member variables end in underscore_
* use dst instead of dist to match pattern use elsewhere in DNNL code.
* Remove workaround that was introduced to handle problems running
convolution based training models. See issue #4762
Signed-off-by: George Nash <george.nash@intel.com>
* Isolate training code and code cleanup
* Do not build if dnnl_gpu_runtime if enable_training is set training code
does not support dnnl_gpu_runtime yet.
* Isolated Training code inside ifdefs so that they wont affect
project if built without training enabled
* Inadvertant changes in whitespace were removed to make code review simpler
* Undid some code reordering that was not needed
* comments added to closing #endif statments to simplify reading complex ifdefs
* Modified the GetPrimitiveDesc functions to return shared_ptr instead of raw
pointer. This matches what was done in Pool code and is safer memory code.
Signed-off-by: George Nash <george.nash@intel.com>
* Address code review issues
- whitespace changes caused by running clang-format on the code
- Several spelling errors fixed
- Removed/changed some ifdefs to improve readability
- other misc. changes in responce to code review.
Signed-off-by: George Nash <george.nash@intel.com>
* Code changes to address code review
- Simplify iteration code using `auto` keyword
- remove C style cast that was not needed
- remove instance variable that was not needed [relugrad.h]
- added the execution providers to `ComputeGradientErrorInternal()`
and `ComputeTheoreticalJacobianTranspose()` instead of using
a pointer to an instance varaible [gradient_checker.h/.cc]
Signed-off-by: George Nash <george.nash@intel.com>
* Combined the default gradient ops test and dnnl gradient ops test for ConvGrad and MaxPoolGrad into one function with the help of a helper function.
This will reduce repeated code.
Signed-off-by: Palangotu Keshava, Chethan's avatarChethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
* Replaced the stack used by convgrad to vector so that the vector(used as stack) can be easily cleared everytime the graph is created.
This will prevent memory leak from convolution kernels being pushed constantly onto the stack.
Signed-off-by: chethan.palangotu.keshava@intel.com
* Code clean up and formating updates
- Removed empty else statment
- updated indentation of code that was causing double curly brackets to look unususal
- Changed check for NumDimensions to Size in Relu and ReluGrad error checking code.
- isolated training code
Signed-off-by: George Nash <george.nash@intel.com>
* Restore inadvertantly removed ConvGrad tests
When combining the DNNL and CPU version of the ConvGrad
tests two test were inadvertantly excluded. This adds
back the Conv3d and Conv3d with strides test cases.
Signed-off-by: George Nash <george.nash@intel.com>
* Add validation to ConvGrad
This validates the dimensions of the ConvGrad match the
passed in Convolution forward primitive description.
The current code for DNNL ConvGrad makes the assumption that the ConvGrad
nodes will be visited in the reverse order from the corresponding Conv nodes
The added validation will return an error if this assumption is not true.
Signed-off-by: George Nash <george.nash@intel.com>
* Do not create new execution providers in provider_test_utils
This removes the code that generated new execution providers in the
OpTester::Run function. This was added because the std::move was
leaving the `entry` value empty so subsequent calls would cause a
segfault.
Problem is this potentially changed the execution_provider because it
would create the default provider dropping any custom arguments.
When the now removed code was originally added the std::move was causing
crashes when the GradientChecker unit tests were run. However, it is no
longer causing problems even with the code removed.
Signed-off-by: George Nash <george.nash@intel.com>
* Change the forward conv stack to a forward conv map
This changes how the forward conv kernel is mapped to the bwd ConvGrad
kernel the problematic stack is no longer used.
The convolution stack made the assumption that the corresponding
ConvGrad operator would be visited in reverse order of the forward
Conv operators. This was always problematic and was unlikely to
work for inception models.
Important changes:
- The weight_name is added to the ConvGrad dnnl_node making it
possible to use the weight_name as a lookup key to find the
Conv forward Kernel
- the `std::vector fwd_conv_stack_` has been replaced with a
`std::map fwd_conv_kernel_map_`
- Although it is not needed lock_guards were added when writing
to and reading from the fwd_conv_kernel_map_ as well as the
fwd_kernel_map_. These should always be accessed by a single
thread when preparing the dnnl subgraphs so the guard should not
be needed but its added just in case.
- Updated the comments ConvGrad.h code to no longer mention the
stack. The error check is not removed. It will be good to verify
there are no errors as we continue to test against more models.
Signed-off-by: George Nash <george.nash@intel.com>
Co-authored-by: Chethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
Co-authored-by: unknown <63478620+jeyblu@users.noreply.github.com>
* Lochi/refactor yolov3 quantization (#6290)
* Refactor the code and move data reader, preprocessing, evaluation to
E2E_example_mode
* Refactor the code.
Move data reader, preprocessing, evaluation to model specific example
under E2E_example_mode
* refactor code
* Move yolov3 example to specific folder and add additional pre/post
processing
* Print a warning message for using newer c_api header on old binary (#6507)
* Fix issues with ArmNN build setup (#6495)
* ArmNN build fixes
* Update BUILD.md to document that the ACL paths must be specified to build ArmNN
* Fix CUDA build error. We don't setup the link libraries correctly/consistently so improve that.
* Fix Windows CI builds by updating test scripts to work with numpy 1.20. (#6518)
* Update onnxruntime_test_python.py to work with numpy 1.20.
Some aliases are deprecated in favor of the built-in python types. See https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
np.array with bytes for entries and dtype of np.void no longer automatically pads. Change a test to adjust for that.
* Fix another test script
* Fix ORTModule branch for orttraining-* pipelines
* Update pytorch nightly version dependency
Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com>
Co-authored-by: George Wu <jywu@microsoft.com>
Co-authored-by: Cecilia Liu <ziyue.liu7@gmail.com>
Co-authored-by: Ryan Hill <38674843+RyanUnderhill@users.noreply.github.com>
Co-authored-by: George Nash <george.nash@intel.com>
Co-authored-by: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com>
Co-authored-by: Yateng Hong <toothache9010@gmail.com>
Co-authored-by: stevenlix <38092805+stevenlix@users.noreply.github.com>
Co-authored-by: Derek Murray <Derek.Murray@microsoft.com>
Co-authored-by: ashbhandare <ash.bhandare@gmail.com>
Co-authored-by: Scott McKay <skottmckay@gmail.com>
Co-authored-by: Changming Sun <chasun@microsoft.com>
Co-authored-by: Tracy Sharpe <42477615+tracysh@users.noreply.github.com>
Co-authored-by: Juliana Franco <jufranc@microsoft.com>
Co-authored-by: Pranav Sharma <prs@microsoft.com>
Co-authored-by: Tixxx <tix@microsoft.com>
Co-authored-by: Jay Rodge <jayrodge@live.com>
Co-authored-by: Du Li <duli1@microsoft.com>
Co-authored-by: Du Li <duli@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Yufeng Li <liyufeng1987@gmail.com>
Co-authored-by: baijumeswani <bmeswani@microsoft.com>
Co-authored-by: Sergii Dymchenko <sedymche@microsoft.com>
Co-authored-by: jingyanwangms <47403504+jingyanwangms@users.noreply.github.com>
Co-authored-by: satyajandhyala <satya.k.jandhyala@gmail.com>
Co-authored-by: S. Manohar Karlapalem <manohar.karlapalem@intel.com>
Co-authored-by: Weixing Zhang <weixingzhang@users.noreply.github.com>
Co-authored-by: Suffian Khan <sukha@microsoft.com>
Co-authored-by: Olivia Jain <oljain@microsoft.com>
Co-authored-by: Chi Lo <54722500+chilo-ms@users.noreply.github.com>
Co-authored-by: Hariharan Seshadri <shariharan91@gmail.com>
Co-authored-by: Ryan Lai <rylai@microsoft.com>
Co-authored-by: Jesse Benson <jesseb@microsoft.com>
Co-authored-by: sfatimar <64512376+sfatimar@users.noreply.github.com>
Co-authored-by: suryasidd <surya.siddharth.pemmaraju@intel.com>
Co-authored-by: sfatimar <sahar.fatima@intel/com>
Co-authored-by: MaajidKhan <n.maajidkhan@gmail.com>
Co-authored-by: mohdansx <mohdx.ansari@intel.com>
Co-authored-by: Xavier Dupré <xadupre@users.noreply.github.com>
Co-authored-by: Michael Goin <mgoin@vols.utk.edu>
Co-authored-by: Michael Giba <michaelgiba@gmail.com>
Co-authored-by: William Tambellini <wtambellini@sdl.com>
Co-authored-by: Hector Li <hecli@microsoft.com>
Co-authored-by: Aishwarya <aibhanda@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: liqunfu <liqfu@microsoft.com>
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: pengwa <pengwa@microsoft.com>
Co-authored-by: Tang, Cheng <souptc@gmail.com>
Co-authored-by: Cheng Tang <chenta@microsoft.com>
Co-authored-by: Tianlei Wu <tlwu@microsoft.com>
Co-authored-by: Ye Wang <52801275+wangyems@users.noreply.github.com>
Co-authored-by: Chun-Wei Chen <jacky82226@gmail.com>
Co-authored-by: Vincent Wang <wangwchpku@outlook.com>
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
Co-authored-by: Luyao Ren <375833274@qq.com>
Co-authored-by: Zhang Lei <zhang.huanning@hotmail.com>
Co-authored-by: Tim Harris <tiharr@microsoft.com>
Co-authored-by: Ashwini Khade <askhade@microsoft.com>
Co-authored-by: Dmitri Smirnov <yuslepukhin@users.noreply.github.com>
Co-authored-by: Alberto Magni <49027342+alberto-magni@users.noreply.github.com>
Co-authored-by: Wei-Sheng Chin <wschin@outlook.com>
Co-authored-by: wezuo <49965641+wezuo@users.noreply.github.com>
Co-authored-by: Jesse Benson <benson.jesse@gmail.com>
Co-authored-by: Wei Zuo <wezuo@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-mgtbby.eastus.cloudapp.azure.com>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-yclzsf.eastus.cloudapp.azure.com>
Co-authored-by: Wenbing Li <10278425+wenbingl@users.noreply.github.com>
Co-authored-by: Martin Man <supermt@gmail.com>
Co-authored-by: M. Zeeshan Siddiqui <mzs@microsoft.com>
Co-authored-by: Ori Levari <ori.levari@microsoft.com>
Co-authored-by: Ori Levari <orlevari@microsoft.com>
Co-authored-by: Ubuntu <OrtTrainingDev3@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sherlock Huang <bahuang@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sheil Kumar <smk2007@gmail.com>
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
Co-authored-by: Ryota Tomioka <ryoto@microsoft.com>
Co-authored-by: Adam Pocock <adam.pocock@oracle.com>
Co-authored-by: Yulong Wang <f.s@qq.com>
Co-authored-by: Faith Xu <faxu@microsoft.com>
Co-authored-by: Xiang Zhang <xianz@microsoft.com>
Co-authored-by: suryasidd <48925384+suryasidd@users.noreply.github.com>
Co-authored-by: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com>
Co-authored-by: Weixing Zhang <wezhan@microsoft.com>
Co-authored-by: Chethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
Co-authored-by: unknown <63478620+jeyblu@users.noreply.github.com>
2021-02-02 16:59:56 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
2022-04-10 05:35:14 +00:00
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ l o n g f o r m e r _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / l o n g f o r m e r /
2024-02-05 18:15:16 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ p h i 2 _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / p h i 2 /
2024-09-18 21:31:59 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ s a m 2 _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / s a m 2 /
2023-02-07 15:49:15 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ s t a b l e _ d i f f u s i o n _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / s t a b l e _ d i f f u s i o n /
2022-04-10 05:35:14 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ t 5 _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / t 5 /
2023-04-26 17:45:52 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ m o d e l s _ w h i s p e r _ s r c }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a n s f o r m e r s / m o d e l s / w h i s p e r /
2019-03-22 18:18:23 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { R E P O _ R O O T } / V E R S I O N _ N U M B E R
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } >
2018-11-20 00:48:22 +00:00
)
2024-07-02 22:37:50 +00:00
if ( onnxruntime_BUILD_SHARED_LIB )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2023-06-17 02:47:09 +00:00
if ( onnxruntime_USE_OPENVINO )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o p e n v i n o _ p y t h o n _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t o o l s /
)
endif ( )
2021-08-28 18:05:21 +00:00
if ( onnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / e x t e r n a l / i n c l u d e /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c r e a t e _ s y m l i n k
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / i n c l u d e / g o o g l e
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / e x t e r n a l / i n c l u d e / g o o g l e
C O M M A N D $ { C M A K E _ C O M M A N D } - E c r e a t e _ s y m l i n k
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / e x t e r n a l / o n n x / o n n x
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / e x t e r n a l / i n c l u d e / o n n x
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y _ d i r e c t o r y
$ { O R T T R A I N I N G _ R O O T } / o r t t r a i n i n g / t e s t / e x t e r n a l _ c u s t o m _ o p s
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / e x t e r n a l _ c u s t o m _ o p s
)
endif ( )
2021-06-09 02:43:59 +00:00
if ( NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD
2024-04-24 01:15:07 +00:00
A N D N O T $ { C M A K E _ S Y S T E M _ N A M E } M A T C H E S " D a r w i n | i O S | v i s i o n O S "
2023-05-21 01:07:39 +00:00
A N D N O T C M A K E _ S Y S T E M _ N A M E S T R E Q U A L " A n d r o i d "
2021-07-28 23:58:13 +00:00
A N D N O T o n n x r u n t i m e _ U S E _ R O C M
2023-05-21 01:07:39 +00:00
A N D N O T C M A K E _ S Y S T E M _ N A M E S T R E Q U A L " E m s c r i p t e n " )
2021-04-23 16:54:09 +00:00
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2021-02-24 04:21:57 +00:00
if ( onnxruntime_BUILD_UNIT_TESTS )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t e s t _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } >
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ q u a n t i z a t i o n _ t e s t _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / q u a n t i z a t i o n /
2021-06-09 02:43:59 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ t e s t _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / t r a n s f o r m e r s /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ t e s t d a t a _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / t r a n s f o r m e r s / t e s t _ d a t a / m o d e l s /
Whisper Model Optimization (#15473)
### Description
This PR contains fusion-level and kernel-level optimizations for
[OpenAI's Whisper](https://github.com/openai/whisper).
Some of the added optimizations include:
- Pruning of duplicate/unnecessary inputs and outputs
- Fusion support for Whisper models with or without these inputs/outputs
(e.g. with these inputs/outputs if exporting with an older official
Optimum version, without these inputs/outputs if exporting with Optimum
from source)
- Attention fusions
- For Whisper's encoder and decoder
- Modified symbolic shape inference for present output when no past
input exists (for decoder)
- Multi-head attention fusions
- For Whisper's decoder and decoder with past
- Packed MatMul for the 3 MatMuls excluded in multi-head attention
fusion
- Attention kernel changes
- CPU:
- Different Q and KV sequence lengths
- Parallel memset for large sequence lengths
- Convert broadcast add after MatMul of Q and K (add_qk) to element-wise
add
- Separate present key-value output into present key and present value
(for multi-head attention spec)
- CUDA:
- Use memory efficient attention compute kernel with present state (for
decoder)
- Multi-head attention kernel changes
- CPU:
- Introduction of multi-head attention CPU kernel (previously did not
exist)
- Use AddBiasReshape instead of AddBiasTranspose when sequence length =
1 (for decoder with past)
- Different Q, K, V input shapes
- Pass past key and past value directly as key and value
- CUDA:
- Use memory efficient attention compute kernel with past and/or present
state (for decoder with past)
### Usage
To use the optimizations, run the ORT transformer optimizer script as
follows:
```
$ cd onnxruntime/onnxruntime/python/tools/transformers/
$ python3 optimizer.py --input <filename>.onnx --output <filename>.onnx --model_type bart --num_heads <number of attention heads, depends on the size of the whisper model used> --hidden_size <attention hidden size, depends on the size of the whisper model used> --use_external_data_format --use_multi_head_attention
```
Once optimized, here's an example of how to run Whisper with [Hugging
Face's Optimum](https://github.com/huggingface/optimum):
```
from transformers.onnx.utils import get_preprocessor
from optimum.onnxruntime import ORTModelForSpeechSeq2Seq
from optimum.pipelines import pipeline as ort_pipeline
import whisper # Installed from OpenAI's repo - setup instructions at https://github.com/openai/whisper/
directory = './whisper_opt' # Where the optimized ONNX models are located
model_name = 'openai/whisper-tiny'
device = 'cpu'
# Get pipeline
processor = get_preprocessor(model_name)
model = ORTModelForSpeechSeq2Seq.from_pretrained(
directory,
use_io_binding=(device == 'cuda'),
provider='CPUExecutionProvider',
).to(device)
pipe = ort_pipeline(
"automatic-speech-recognition",
model=model,
tokenizer=processor.tokenizer,
feature_extractor=processor.feature_extractor,
device=(-1 if device == 'cpu' else 0),
)
# Load audio file and run pipeline
audio = whisper.load_audio('tests/jfk.flac')
audio = whisper.pad_or_trim(audio)
outputs = pipe([audio])
print(outputs)
```
Note: In order to use these changes with Optimum, it is recommended to
use Optimum from source to have the following changes:
- https://github.com/huggingface/optimum/pull/872
- https://github.com/huggingface/optimum/pull/920
### Motivation and Context
This PR helps the following issues:
- https://github.com/microsoft/onnxruntime/issues/15100
- https://github.com/microsoft/onnxruntime/issues/15235
- https://github.com/huggingface/optimum/issues/869 (work in progress)
This PR can be used with the other currently merged Whisper PRs:
- https://github.com/microsoft/onnxruntime/pull/15247
- https://github.com/microsoft/onnxruntime/pull/15339
- https://github.com/microsoft/onnxruntime/pull/15362
- https://github.com/microsoft/onnxruntime/pull/15365
- https://github.com/microsoft/onnxruntime/pull/15427
This PR uses changes from the following merged PRs:
- https://github.com/microsoft/onnxruntime/pull/14198
- https://github.com/microsoft/onnxruntime/pull/14146
- https://github.com/microsoft/onnxruntime/pull/14201
- https://github.com/microsoft/onnxruntime/pull/14928 (this introduced
the new multi-head attention spec)
2023-04-19 00:13:54 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ t e s t d a t a _ w h i s p e r }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / t r a n s f o r m e r s / t e s t _ d a t a / m o d e l s / w h i s p e r /
2023-11-19 07:39:04 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ t r a n s f o r m e r s _ t e s t d a t a _ c o n f o r m e r }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / t r a n s f o r m e r s / t e s t _ d a t a / m o d e l s / c o n f o r m e r /
2021-02-24 04:21:57 +00:00
)
endif ( )
2021-08-06 15:30:27 +00:00
if ( onnxruntime_BUILD_UNIT_TESTS AND onnxruntime_ENABLE_EAGER_MODE )
file ( GLOB onnxruntime_eager_test_srcs CONFIGURE_DEPENDS
" $ { O R T T R A I N I N G _ R O O T } / o r t t r a i n i n g / e a g e r / t e s t / * . p y "
)
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ e a g e r _ t e s t _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / e a g e r _ t e s t /
)
endif ( )
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
if ( onnxruntime_ENABLE_TRAINING )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
2021-02-24 04:21:57 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / a m p
2022-02-18 22:00:49 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / e x p e r i m e n t a l
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / e x p e r i m e n t a l / g r a d i e n t _ g r a p h
2021-02-24 04:21:57 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o p t i m
2021-04-26 21:53:50 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e
2021-07-30 20:05:32 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / j s o n _ c o n f i g
2021-09-28 00:18:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / h i e r a r c h i c a l _ o r t m o d u l e
2021-06-29 01:11:58 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s
2021-09-30 14:37:35 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c p u / a t e n _ o p _ e x e c u t o r
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c p u / t o r c h _ i n t e r o p _ u t i l s
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c u d a / t o r c h _ g p u _ a l l o c a t o r
2021-10-26 05:13:49 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c u d a / f u s e d _ o p s
2023-10-27 02:29:27 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / g r a p h _ o p t i m i z e r s
2024-04-18 18:30:15 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / p i p e
2023-07-13 10:17:58 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t _ t r i t o n
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t _ t r i t o n / k e r n e l
2023-08-04 05:58:21 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / u t i l s
2022-02-14 21:46:14 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / u t i l s / d a t a /
Statistics tool for ORTModule convergence parity (#15020)
### Statistics tool for ORTModule convergence parity
As ORTModule get more and more validated, it is pretty fast to
intergrade PyTorch based model with ORT.
The same time, we need make sure once there is convergence issue, we
don't spend months of time to investigate. As part of this efforts, this
PR is introducing a tool to dump activation statistics without much
involvement from users. The dumping results contains only some statistic
numbers plus sampled data, which is not big, compared with dumping all
the tensors, it is much faster and space efficient.
For us to use it, two single lines are needed before wrapping ORTModule.
For baseline run, need also apply the same trick.
```
+ from onnxruntime.training.utils.hooks import SubscriberManager, StatisticsSubscriber
+ SubscriberManager.subscribe(model, [StatisticsSubscriber("pt_out", override_output_dir=True)])
```
Once you run the steps, following command can be used to merge result
into per-step-summary respectively for ORT and baseline runs.
```bash
python -m onnxruntime.training.utils.hooks.merge_activation_summary --pt_dir pt_out --ort_dir ort_out --output_dir /tmp/output
```
Docs is added here as part of this PR [convergence investigation
notes](https://github.com/microsoft/onnxruntime/blob/pengwa/conv_tool/docs/ORTModule_Convergence_Notes.md)
Based on the generated merged files, we can compare them with tools.
### Design and Implementation
This PR introduced a common mechanism registering custom logic for
nn.Module's post forward hooks. And statistics for activation
(StatisticsSubscriber) is one of the implementations. If there is other
needs, we can define another XXSubscriber to do the customized things.
2023-03-23 12:34:24 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / u t i l s / h o o k s /
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ r o o t _ s r c s }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g /
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ a m p _ s r c s }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / a m p /
2022-02-18 22:00:49 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ e x p e r i m e n t a l _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / e x p e r i m e n t a l /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ g r a d i e n t _ g r a p h _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / e x p e r i m e n t a l / g r a d i e n t _ g r a p h /
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o p t i m _ s r c s }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o p t i m /
2021-04-26 21:53:50 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e /
2021-07-30 20:05:32 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ e x p e r i m e n t a l _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ e x p e r i m e n t a l _ j s o n _ c o n f i g _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / j s o n _ c o n f i g /
2021-09-28 00:18:22 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ e x p e r i m e n t a l _ h i e r a r c h i c a l _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / h i e r a r c h i c a l _ o r t m o d u l e /
2021-06-29 01:11:58 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ t o r c h _ c p p _ e x t _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ t o r c h _ c p p _ e x t _ a t e n _ o p _ e x e c u t o r _ s r c s }
2021-09-30 14:37:35 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c p u / a t e n _ o p _ e x e c u t o r /
2021-09-01 01:29:26 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ t o r c h _ c p p _ e x t _ t o r c h _ i n t e r o p _ u t i l s _ s r c s }
2021-09-30 14:37:35 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c p u / t o r c h _ i n t e r o p _ u t i l s /
2021-06-29 01:11:58 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ t o r c h _ c p p _ e x t _ t o r c h _ g p u _ a l l o c a t o r _ s r c s }
2021-09-30 14:37:35 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c u d a / t o r c h _ g p u _ a l l o c a t o r /
2021-10-06 03:50:34 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
2021-10-26 05:13:49 +00:00
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ t o r c h _ c p p _ e x t _ f u s e d _ o p s _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / t o r c h _ c p p _ e x t e n s i o n s / c u d a / f u s e d _ o p s /
2023-10-27 02:29:27 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ g r a p h _ o p t i m i z e r s _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / g r a p h _ o p t i m i z e r s /
2024-04-18 18:30:15 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t m o d u l e _ p i p e _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t m o d u l e / e x p e r i m e n t a l / p i p e /
2023-07-13 10:17:58 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t _ t r i t o n _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t _ t r i t o n /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o r t _ t r i t o n _ k e r n e l _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o r t _ t r i t o n / k e r n e l /
2023-08-04 05:58:21 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ u t i l s _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / u t i l s /
2022-02-14 21:46:14 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ u t i l s _ d a t a _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / u t i l s / d a t a /
Statistics tool for ORTModule convergence parity (#15020)
### Statistics tool for ORTModule convergence parity
As ORTModule get more and more validated, it is pretty fast to
intergrade PyTorch based model with ORT.
The same time, we need make sure once there is convergence issue, we
don't spend months of time to investigate. As part of this efforts, this
PR is introducing a tool to dump activation statistics without much
involvement from users. The dumping results contains only some statistic
numbers plus sampled data, which is not big, compared with dumping all
the tensors, it is much faster and space efficient.
For us to use it, two single lines are needed before wrapping ORTModule.
For baseline run, need also apply the same trick.
```
+ from onnxruntime.training.utils.hooks import SubscriberManager, StatisticsSubscriber
+ SubscriberManager.subscribe(model, [StatisticsSubscriber("pt_out", override_output_dir=True)])
```
Once you run the steps, following command can be used to merge result
into per-step-summary respectively for ORT and baseline runs.
```bash
python -m onnxruntime.training.utils.hooks.merge_activation_summary --pt_dir pt_out --ort_dir ort_out --output_dir /tmp/output
```
Docs is added here as part of this PR [convergence investigation
notes](https://github.com/microsoft/onnxruntime/blob/pengwa/conv_tool/docs/ORTModule_Convergence_Notes.md)
Based on the generated merged files, we can compare them with tools.
### Design and Implementation
This PR introduced a common mechanism registering custom logic for
nn.Module's post forward hooks. And statistics for activation
(StatisticsSubscriber) is one of the implementations. If there is other
needs, we can define another XXSubscriber to do the customized things.
2023-03-23 12:34:24 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ u t i l s _ h o o k s _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / u t i l s / h o o k s /
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
)
2023-01-03 21:28:16 +00:00
if ( onnxruntime_ENABLE_TRAINING_APIS )
2022-05-25 01:21:39 +00:00
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o n n x b l o c k
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o n n x b l o c k / l o s s
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o n n x b l o c k / o p t i m
2022-09-16 16:38:24 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E m a k e _ d i r e c t o r y $ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / a p i
2022-05-25 01:21:39 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o n n x b l o c k _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o n n x b l o c k /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o n n x b l o c k _ l o s s _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o n n x b l o c k / l o s s /
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o n n x b l o c k _ o p t i m _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / o n n x b l o c k / o p t i m /
2022-09-16 16:38:24 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ a p i _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / t r a i n i n g / a p i /
2022-05-25 01:21:39 +00:00
)
endif ( )
Add new PytTrch front-end (#4815)
* Add ORTTrainerOptions class for the new pytorch frontend (#4382)
Add ORTTrainerOptions class and some placeholders
* Add _ORTTrainerModelDesc to perform validation for model description (#4416)
* Add Loss Scaler classes to the new frontend (#4306)
* Add TrainStepInfo used on the new frontend API (#4256)
* Add Optimizer classes to the new frontend (#4280)
* Add LRScheduler implementation (#4357)
* Add basic ORTTrainer API (#4435)
This PR presents the public API for ORTTrainer for the short term
development.
It also validates and saves input parameters, which will be used in the
next stages, such as building ONNX model, post processing the model and
configuring the training session
* Add opset_version into ORTTrainerOptions and change type of ORTTrainer.loss_fn (#4592)
* Update ModelDescription and minor fix on ORTTrainer ctor (#4605)
* Update ModelDescription and minor fix on ORTTrainer/ORTTrainerOptions
This PR keeps the public API intact, but changes how model description is stored on the backend
Currently, users creates a dict with two lists of tuples.
One list called 'inputs' and each tuple has the following format tuple(name, shape).
The second list is called 'outputs' and each tuple can be either tuple(name, shape) or tuple(name, shape, is_loss).
With this PR, when this dict is passed in to ORTTrainer, it is fully validated as usual.
However, tuples are internally replaced by namedtuples and all output tuples will have
tuple(name, shape, is_loss) format instead of is_loss being optionally present.
Additionally to that normalization in the internal representation (which eases coding),
two internal methods were created to replace a namedtuple(name, shape) to namedtuple(name, shape, dtype)
or namedtuple(name, shape, is_loss, dtype) dependeing whether the tuple is an input or output.
This is necessary as ORTTRainer finds out data types of each input/output during model export to onnx.
Finally, a minor fix was done on ORTTrainer. It could initialize ORTTrainerOptions incorrectly when options=None
* Rename input name for test
* Add ONNX Model Export to New Frontend (#4612)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Create training session + minor improvements (#4668)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Save ONNX model in file (#4671)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add eval step (#4674)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add train_step (#4677)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add LR Scheduler (#4694)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add deterministic compute tests (#4716)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add legacy vs experimental ORTTrainer accuracy comparison (#4727)
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add Mixed precision/LossScaler + several fixes (#4739)
Additionally to the mixed precision/loss scaler code, this PR includes:
* Fix CUDA training
* Add optimization_step into TrainStepInfo class
* Refactor LRSCheduler to use optimization_step instead of step
* Updated several default values at ORTTrainerOptions
* Add initial Gradient Accumulation supported. Untested
* Fix ONNX model post processing
* Refactor unit tests
* Add ONNX BERT example + minor fixes (#4757)
* Fix training issue when passing ONNX file into ORTTrainer
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add Dynamic Shape support (#4758)
* Update DeepSpeed Zero Stage option to a separate option group (#4772)
* Add support to fetches (#4777)
* Add Gradient Accumulation Steps support (#4793)
* Fix Dynamic Axes feature and add unit test (#4795)
* Add frozen weights test (#4807)
* Move new pytorch front-end to 'experimental' namespace (#4814)
* Fix build
Co-authored-by: Rayan-Krishnan <rayankrishnan@live.com>
Co-authored-by: Rayan Krishnan <t-rakr@OrtDevTest2v100.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
2020-08-17 16:45:25 +00:00
endif ( )
2019-12-03 15:34:23 +00:00
if ( onnxruntime_USE_DNNL )
2018-11-20 00:48:22 +00:00
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
2020-05-09 00:11:29 +00:00
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { D N N L _ D L L _ P A T H } $ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ d n n l >
2020-08-11 04:17:16 +00:00
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
2018-11-20 00:48:22 +00:00
)
endif ( )
2019-04-21 00:02:35 +00:00
2024-02-01 05:08:26 +00:00
if ( onnxruntime_USE_VITISAI )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { D N N L _ D L L _ P A T H } $ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ v i t i s a i >
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2020-08-13 22:24:44 +00:00
if ( onnxruntime_USE_TENSORRT )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
Sync ORTModule branch with master and fix tests (#6526)
* Deprecate Python global configuration functions [Part 1] (#5923)
Enable options to be set via execution provider (EP)-specific options and log deprecation warning from current global configuration functions.
* remove dnnl_dll_path from post build copy (#6142)
* Model Fusion For Bart (#6105)
Fusion fix for Bart models
* Unify IExecutionProvider and IExecutionProviderFactory interfaces (#6108)
* Remove Provider_IExecutionProvider and make the internal IExecutionProvider usable by shared providers
* Change Provider_IExecutionProviderFactory to be the core version.
* Enable running the mnist_training sample without cuda (#6085)
Signed-off-by: George Nash <george.nash@intel.com>
* nnapi add min max support (#6117)
* Fix CUDA test hang: (#6138)
- Make condition check in `CUDAAllocatorTest` to ensure CUDA device is present.
* Fix TensorRT kernel conflict issue for subgraphs of control flow operators (#6115)
* add static subgraph kernel index
* change kernel naming to avoid conflicts
* Add gradient registration for Abs. (#6139)
* Partition initial optimizer state for Zero-1 (#6093)
* Initial changes
* Working changes
* Working changes
* Cleanup
* fix windows CI
* Review comments
* review comments
* Fix edge case in BFCArena where allocation failures could lead to an infinite loop. (#6145)
#4656
* Revert "work around of the build break in mac (#6069)" (#6150)
This reverts commit 3cae28699bed5de1fcaadb219fa69bae0fc3cee8.
* Fix clean_docker_image_cache.py detection of image pushes. (#6151)
Fix clean_docker_image_cache.py detection of image pushes. They were being ignored because the expected HTTP status code was wrong. For pushes, it's 201 instead of 200.
* MLAS: add NEON version of int8 depthwise convolution (#6152)
* Using a map of of ops to stages as input of partition function. (#5940)
* New partition algorithm running before AD
* Convert cut_group_info into device map. Work in progress -- works for bert-tiny with pp=2
* Removing code for partition of bwd graphs
* Remove old code
* Adding some verification code
* Handle Shared Initializer
* Renaming rank with stage
* Added first unit test
* new test
* redundant check
* undo change in bert
* Moved cut-based partition to testing utils file
Co-authored-by: xzhu1900
Co-authored-by: wschin
* New conversion function and tests
* minor
* remove test that is not needed2
* improve GetDeviceAssignment and PR comments
* minor changes
* PR comments
* improving documentation and variable naming
* add documentation
* Variable naming and docs
* more doc improvements
* more doc improvements
* missing static cast
* Fix test file for windows
* Fix test file for windows
* Fix test file for windows
* stage id is not the same as rank id
* PR comments
* PR comments
* More comments
* More comments
* Minor fix to satisfy c++14 (#6162)
* Deprecating Horovod and refactored Adasum computations (#5468)
deprecated horovod submodule
refactored adasum logic to be ort-native
added tests for native kernel and e2e tests
* Update TensorRT-ExecutionProvider.md (#6161)
* Bugfix for topk cuda kernel (#6164)
* fix the issue that std::numeric_limits cannot handle half type
* adding a test
Co-authored-by: Du Li <duli@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Revert "Fuse MatMulIntegerToFloat only when scales are scalar (#6008)" (#6169)
This reverts commit f2dcba7afe0d42ebdaaef0c6cdf913a1156c9e98.
* Remove ignored build warnings for pybind on Mac (#6165)
* save_checkpoint, load_checkpoint and aggregate_checkpoints (#6136)
* save_checkpoint and load_checkpoint implementations
* checkpoint aggregation logic
* unit tests for save_checkpoint, load_checkpoint and aggregate_checkpoints
* Don't try to bind unused inputs in the Training frontend (#6166)
* Update documentation for contributing a PR and add deprecation notices for PyOp and ORT server. (#6172)
* aggregate model states only for the case when mixed precision was true (#6176)
* [NNAPI EP] Enable per-channel quantization for QlinearConv (#6155)
* Enable qlinearconv per-channel quantization
* Fix the android CI test failure
* Add Android Version Check for Per-Channel Quant
* Address PR comments
* Fix some minor issues
* Add verification of per-channel zero points
* Make the error tolerance configurable
* Fix typo in BERT pretraining script (#6175)
A misplaced `}` meant that the `'enable_adasum'` option was interpreted incorrectly, causing the test to fail.
* Update get_docker_image.py to enable use without image cache container registry. (#6177)
Update get_docker_image.py to enable use without image cache container registry.
* Helper for compiling EP to generate deterministic unique ids for use in MetaDef names (#6156)
* Create a helper for generating unique ids that can be used by an EP that creates compiled nodes and needs ids to be deterministic for a model when used in multiple sessions.
Added to IExecutionProvider as this can potentially be used by all compiling EPs and is more robust than a simplistic counter (although EP implementer is free to choose either approach).
* Restructure the helper so it can be called across the EP bridge.
Add ability to call id generation helper from EP bridge
- convert DNNL EP to use helper to validate
Address issue where a new Model may be loaded into the same address as a previous one.
- hash the bytes in the Graph instance (1728 bytes currently) to use as the key to the full hash for the model
Add lock around id generation to ensure no issues if multiple sessions partitions graphs at exactly the same time.
- Extremely unlikely but would be hard to debug and the locking cost is not an issue as it's only incurred during graph partitioning and not execution.
* Backend APIs for checkpointing (#5803)
* Add backend API GetOptimizerState and GetModelState
* add GetPartitionInfoMap
* Android coverage dashboard (#6163)
* Write the report to a file.
* Post code coverage to the Dashboard database.
* Add usage details of unified MCR container image (#6182)
Going forward, a single unifed docker image will be published in
MCR. The hardware accelerator target choice will have to be made
in the application using OpenVINO EP's runtime config options.
* improve perf for softmax (#6128)
* improve perf for both gathergrad and softmax
* revert the change in gathergrad and will be done in another PR.
* address comments from code review.
* Tune fast Gelu to use exp(x) instead of tanh(x) on Rocm platform (#6174)
* tune fast gelu to use exp(x) instead of tanh(x) on rocm
* update to use expression 2/(1+exp(-2x))-1 for stability
* Add Status.csv to EP Perf Tool (#6167)
* merge master, keep postprocess status commit
* download float16.py everytime
* removing hardcoded values
* Lochi/quantization tool for trt (#6103)
* Initial implementation of generating calibration dynamic range table
* Initialize validation support for Quantization
* Initialize validation support for Quantization (cont.)
* Improve validation support for Quantization
* Improve validation support for Quantization
* Rewrite/Refine for calibration and validation
* Rewrite/Refine for calibration and validation (cont.)
* Refine code
* Refine code
* Add data reader for BERT
* Add flatbuffers to serialize calibration table
* Refine code and add BERT evaluation
* Refine the code
* minor modification
* Add preprocess/postprocess of vision team yolov3 and refine the code
* Update annotation
* Make bbox cooridates more accurate
* Fix bug
* Add support of batch processing
* Batch processing for model zoo yolov3
* Add batch inference for evaluation
* Refine the code
* Add README
* Add comments
* Refine the code for PR
* Remove batch support checking in data_reader and refine the code
* Refine the code for PR
* Refine the code for PR review
Co-authored-by: Olivia Jain <oljain@microsoft.com>
* Implement ScatterND for CUDA EP (#6184)
* Condition fix in Resize operator (#6193)
* Clean up checkpoint tests to use the new checkpoint functions (#6188)
* add deprecation warning for old checkpoint functions
* update all the distributed checkpoint tests to use new checkpoint functions
* Implement comparing outputs that are sequence of maps of strings to floats (#6180)
* Implement conversion from ortvalue to Itensor for string tensors and comparing sequence of maps of strings to floats
* PR comments
* Dockerfile to build onnxruntime with ROCm 4.0
* Add ability to skip GPU tests based on GPU adapter name (#6198)
* Implement conversion from ortvalue to Itensor for string tensors and comparing sequence of maps of strings to floats
* PR comments
* Add ability to skip gpu tests according to adapter description
* spacing
* spacing
* spacing
* Openvino ep 2021.2 (#6196)
* Enabling fasterrcnn variant and vehicle detector
* changes for 2021_2 branch
* yolov3_pytorch commit
* fixed braces in basic_backend.cc
* ci information added
* faster rcnn variant and vehicle detector changes were made in 2021.1 and not in 2021.2
* some changes to support unit tests
* disable some tests which are failing
* fix myriad tests for vehicle detector
* Did some cleanup
*cleaned up comments
*Disabled Add_Broadcast_0x1 and Add_Broadcast_1x0
tests on MYRIAD_FP16 backend due to a bug
*cleaned up capability_2021_2.cc file
*Removed extra conditions which were added
for some validation in backend_utils
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* yolov3 pytorch workaround to ensure that the output names are matched
* gemmoptest fixed on myriad
* Fixed MYRIADX CPP Test Failures
*Expand,GatherND,Range,Round op's
are only supported in model
*where op with float input data
types are not supported and fixed
*Scatter and ScatterElements op's with
negative axis are fixed
*Reshape op with 0 dim value are not
supported and fixed
*Disabled InstanceNorm_2 test on MYRIADX
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* make changes to yolov3 pytorch
* Fixed python unit tests
*Fixed failing python tests on vpu,
GPU and CPU
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Fixes POW op failures on GPU_FP16
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Clean up capability_2021_2.cc
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Updated docx for MultiThreading option
*Added extra info on setting the num_of_threads
option using the API and it's actual usage
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* fixed slice and removed extra prints
* Disabled failing python tests
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Minor changes added in capabilty_2021_2
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* made changes to slice to avoid failures
* Disabling FP16 support for GPU_FP32
->Inferencing an FP16 model on GPU_FP32
leads to accuracy mismatches. so, we would
rather use GPU_FP16 to infer an FP16 model
on GPU Device
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* Updated docx for Inferencing a FP16 Model
Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
* fix for mask rcnn
* Script for installing openvino from source
* Updated with openvino 2021.2 online installation
* code comment fixes
fixed accuracy mismatch for div
* Update OpenvinoEP-ExecutionProvider.md
updated for 2021.2 branch
* Update README.md
updated dockerfile documentation
* Update BUILD.md
build.md update documentation
* permissiong change of install_openvino.sh
* made changes to align with microsoft onnxruntime changes
* Updated with ov 2021.2.200
Co-authored-by: suryasidd <surya.siddharth.pemmaraju@intel.com>
Co-authored-by: sfatimar <sahar.fatima@intel/com>
Co-authored-by: MaajidKhan <n.maajidkhan@gmail.com>
Co-authored-by: mohdansx <mohdx.ansari@intel.com>
* Fix a memory leak in test_inference.cc (#6201)
* Fix a memory leak in test_inference.cc
* Use TArray in AMD element-wise kernels, rather than manually copying memory to device.
* Remove most ROCm-specific element-wise code and reuse CUDA element-wise code.
* Minor change to improve performance for operator Pad. (#5537)
* small improvment for pad
* Support double for operators Log, Reciprocal, Sum (CPU) (#6032)
* Support double for operators Log, Reciprocal, Sum
* remove tesdt erf_double
* Support double for operators Where, LpNormalisation (#6034)
* Support double for operators Relu, Tanh, Sigmoid (#6221)
* Fix ImportError in build.py (#6231)
There is a possible ImportError where build.py can import the wrong 'util' package if there are others present in `sys.path` already
* Removed executor todo that looks dead. (#6234)
* Remove MKLML/openblas/jemalloc build config (#6212)
* Remove python 3.5
* Update the readme file
* Upgrade build.py to assert for python 3.6+
Upgrade build.py to assert for python 3.6+
as python 3.5 cannot build anymore todays master.
* Support MLFloat16 type in Pow opset-12 CUDA kernel (#6233)
* MLAS: handle MlasGemm(M/N/K==0) cases (#6238)
* Support double for operator TopK + fix one bug in TopK implementation for GPU for double (#6220)
* Support double for operator TopK
* add static classes for topk/double
* fix cast issue in topk
* Support double for operator Gemm + fix bug in gemm implementation for cuda, rocm when sizeof(type) != sizeof(float) (#6223)
* Support double for operator Gemm
* fix type size while copying data in gemm operator for GPU
* fix type in gemm implementation for rocm
* Support double for operator ReduceMean, ReduceLogSumExp (#6217)
* Support double for operators ReduceMean, ReduceLogSumExp
* Support double for operator ArgMin (#6222)
* Support double for operator ArgMin
* add test specifically for double
* add new test on pai-excluded-tests.txt
* Update BUILD.md
* Update manylinux docker image to the latest (#6242)
* Fix allocator issue for TensorRT IOBinding (#6240)
* Fix issue: https://github.com/microsoft/onnxruntime/issues/6094
Root cause: we didn't expose the OrtMemoryInfo for TRT, so it will cause issue if user want use IObinding for Tensorrt.
Short term fix, add the OrtMemoryInfo for TRT. Long term should unify the allocator for CUDA and TRT
* Tune BiasGeluGradDx kernel in approximation mode to avoid tanh(...) on Rocm (#6239)
* bias gelu grad use exp(...) instead
* update cuda to rocm
* missing semicolon
* comment
* remove dockerfile
* missing factor of two
* Refactor EP Perf Tool (#6202)
* merge master, keep postprocess status commit
* download float16.py everytime
* using variables to reference eps
* adding ACL EP to ep perf tool
* accuracy with absolute tolerance configurable
* add acl to dict + remove commented line
* Documentation for distributed CI tests pipeline (#6140)
* Remove a debug log in provider_test_utils.cc (#6200)
* Add the Concat Slice Elimination transform, fix constant_folding transform (#5457)
* Add concat slice transform + test
* Cosmetic improvements in concat slice transform
* Remove unrelated file, fix comment, fix constant folding bug
* Add test onnx graph
* fix windows build
* Review comments
* review comment
Co-authored-by: Aishwarya <aibhanda@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Add MakeStringLite which uses current locale, update some MakeString call sites to use it instead. (#6252)
* Add MakeStringLite which uses current locale, update macros to use that to generate messages.
* Convert calls to MakeStringLite().
* Liqun/speech model loop to scan (#6070)
Provide a tool to convert Loop to Scan for Nuphar performance
Fix Nuphar CI pipeline failures.
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* model parallel refinement (#6244)
* Megatron Transformation as a seperate step
* remove useless header
* clang formating
* Re-Structure megatron transformer for subsquent changes
* fix comments
* Allow querying a GraphProto's doc_string as part of ModelMetadata (#6248)
* Fix Linux/Mac error message on input type mismatch (#6256)
* add bfloat16 to gathergrad type constrains (#6267)
Co-authored-by: Cheng Tang <chenta@microsoft.com>
* Fix VS 2017 build break (#6276)
* Deprecate Python global configuration functions [Part 2] (#6171)
Update Python API to allow more flexibility for setting providers and provider options.
The providers argument (InferenceSession/TrainingSession constructors, InferenceSession.set_providers()) now also accepts a tuple of (name, options dict).
Fix get_available_providers() API (and the corresponding function in the C API) to return the providers in default priority order. Now it can be used as a starting point for the providers argument and maintain the default priority order.
Convert some usages of the deprecated global configuration functions to use EP-specific options instead.
Update some EP-specific option parsing to fail on unknown options.
Other clean up.
* Add script to preprocess python documentation before publishing (#6129)
* add script to preprocessing python documentation before publishing
* rename past to past_key_values for GPT-2 (#6269)
rename past to past_key_values for transformers 4.*
* Rename MakeString and ParseString functions. (#6272)
Rename MakeString to MakeStringWithClassicLocale, MakeStringLite to MakeString, *ParseString to *ParseStringWithClassicLocale.
Add missing pass-through versions of MakeStringWithClassicLocale for string types.
* Increase timeout for Linux GPU CUDA11 build. (#6280)
* Add helper to compare model with different precision (#6270)
* add parity_check_helper.py
* add real example
* remove lines
* Fix Min/Max CPU kernels for float16 type (#6205)
* fix data_ptr assertion error for past_sequence_length=0 in GPT-2 (#6284)
fix io binding crash for past_sequence_length=0
* A list of changes in transformers tool (#6224)
* longformer fp16 e2e
* add fp16/fp32 parity check helper file
* excludes nodes with subgraph in profiling
* use onnxconverter_common to do fp32->fp16
* add version check for onnxconverter_common
* remove helper file
* add pkg installation on notebooks and script
* Workaround for static_cast<double>(half)
* Add workaround to remove ROCm-specific binary-elementwise files.
* Update nuget build (#6297)
1. Update the ProtoSrc path. The old one is not used anymore.
2. Regenerate OnnxMl.cs
3. Delete some unused code in tools/ci_build/build.py
4. Avoid set intra_op_param.thread_pool_size in ModelTests in OpenMP build.
5. Fix a typo in the C API pipeline.
* Enable ONNX backend test of SequenceProto input/output (#6043)
* assert sequence tensor and remove skips
* update testdata json
* use ONNX 1.8 in cgmanifest.json
* use previous commit to workaround
* update ONNX commit ID in docker
* skip test_maxpool_2d_dilations test for now
* update function name
* add --sequence_lengths option (#6285)
* more dtype for Equal CUDA kernel (#6288)
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
* Force reinstall onnx python package on Windows (#6309)
* update transformers required package versions (#6315)
* Remove abs in LpPool (#6303)
* Support 1D input for Conv + Mul/Add fusion optimizer with test (#6295)
* Support 1D input (N C H) for Conv + Mul/Add fusion optimizer with test cases and test models.
* Add longformer to python package (#6314)
* add longformer to python package
* move test related script and data to a new folder
* Avoid false sharing on thread pool data structures (#6298)
Description: This change adds alignment and padding to avoid false sharing on fields in the thread pool. It also adds a new microbenchmark to profile thread-pool performance over short loops.
Motivation and Context
MobileNet on a 2*12-core system showed a performance gap between the ORT thread pool and OpenMP. One cause appeared to be false sharing on fields in the thread pool: ThreadPoolParallelSection::tasks_finished (which the main thread spins on waiting for workers to complete a loop), and the RunQueue::front_ and back_ fields (used respectively by the worker thread and the main thread).
The additional micro-benchmark BM_ThreadPoolSimpleParallelFor tests performance of loops of different sizes at different thread counts. The results below are on a machine with 2*14-core processors (E5-2690 v4) running with 1, 14, 15, and 28 threads. For each test, the microbenchmark has N threads run a loop with N iterations; hence a perfect result is for the time taken to be constant as additional threads are added (although we will also see power management effects helping at very low thread counts). The loop durations (100000, 10000, 1000) correspond roughly to 200us, 20us, and 2us on this machine.
Before change:
BM_ThreadPoolSimpleParallelFor/1/1/100000/real_time 17153 us 17154 us 32
BM_ThreadPoolSimpleParallelFor/14/14/100000/real_time 22553 us 22553 us 30
BM_ThreadPoolSimpleParallelFor/15/15/100000/real_time 21521 us 21521 us 29
BM_ThreadPoolSimpleParallelFor/28/28/100000/real_time 24111 us 24111 us 24
BM_ThreadPoolSimpleParallelFor/1/1/10000/real_time 1719 us 1719 us 407
BM_ThreadPoolSimpleParallelFor/14/14/10000/real_time 3409 us 3409 us 200
BM_ThreadPoolSimpleParallelFor/15/15/10000/real_time 3541 us 3541 us 201
BM_ThreadPoolSimpleParallelFor/28/28/10000/real_time 4576 us 4576 us 151
BM_ThreadPoolSimpleParallelFor/1/1/1000/real_time 174 us 174 us 4017
BM_ThreadPoolSimpleParallelFor/14/14/1000/real_time 1586 us 1586 us 402
BM_ThreadPoolSimpleParallelFor/15/15/1000/real_time 1586 us 1586 us 397
BM_ThreadPoolSimpleParallelFor/28/28/1000/real_time 2864 us 2864 us 232
After change:
BM_ThreadPoolSimpleParallelFor/1/1/100000/real_time 17160 us 17160 us 33
BM_ThreadPoolSimpleParallelFor/14/14/100000/real_time 20989 us 20989 us 31
BM_ThreadPoolSimpleParallelFor/15/15/100000/real_time 22286 us 22286 us 31
BM_ThreadPoolSimpleParallelFor/28/28/100000/real_time 24631 us 24631 us 25
BM_ThreadPoolSimpleParallelFor/1/1/10000/real_time 1718 us 1718 us 407
BM_ThreadPoolSimpleParallelFor/14/14/10000/real_time 2868 us 2868 us 242
BM_ThreadPoolSimpleParallelFor/15/15/10000/real_time 2907 us 2907 us 240
BM_ThreadPoolSimpleParallelFor/28/28/10000/real_time 3872 us 3872 us 186
BM_ThreadPoolSimpleParallelFor/1/1/1000/real_time 175 us 175 us 3938
BM_ThreadPoolSimpleParallelFor/14/14/1000/real_time 933 us 933 us 659
BM_ThreadPoolSimpleParallelFor/15/15/1000/real_time 912 us 912 us 591
BM_ThreadPoolSimpleParallelFor/28/28/1000/real_time 1976 us 1976 us 317
* fix opset imports for function body (#6287)
* fix function opsets
* add tests and update onnx
* changes per review comments
* add comments
* plus updates
* build fix
* Remove false positive prefast warning from threadpool (#6324)
* Java: add Semmle to Java publishing pipelines (#6326)
Add Semmle to Java API pipeline
Add security results publishing and add Java GPU.
* Quantization support for split operator with its NHWC support (#6107)
* Make split working for quantization.
* NHWC transformer support for split operator
* Refactor some according to Feedback. Will add test cases soon.
* Fix build error on windows.
* Add test case for split op on uint8_t support
* Add nhwc_transformer_test for split uint8_t support
* Some change according to PR feedbacks.
* Liqun/enable pipeline parallel test (#6331)
enable pipeline parallel test
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Use onnxruntime_USE_FULL_PROTOBUF=OFF for the cuda execution provider (#6340)
This removes a special case of the cuda EP.
* MLAS: add fallback implementation for quantized GEMM (#6335)
Add a non-vectorized version of the kernel used for the quantized version of MlasGemm.
* Delete float16.py (#6336)
No longer needed. Also doesn't pass policheck.
* Enable add + softmax fusion for Rocm platform (#6259)
* add bias softmax; tests appear to pass
* check fusion occurs for rocm as well
* check for rocm provider compatible as well
* build for cpu scenario as well
* try again; broader cope
* proper scope on kGpuExecutionProvider
* been editing wrong file
* remove commented #include lines
* try again due to mac os ci error
* try again
* test fusion both cuda and rocm to avoid mac ci error
* add external data support to tensor proto utils (#6257)
* update unpack tensor utilities to support loading external data
* more updates
* fix test
* fix nuphar build
* minor build fix
* add tests
* fix Android CI
* fix warning
* fix DML build failure and some warnings
* more updates
* more updates
* plus few updates
* plus some refactoring
* changes per review
* plus some change
* remove temp code
* plus updates to safeint usage
* build fix
* fix for safeint
* changed wording. (#6337)
* Remove OpSchema dummy definition. Only needed for Function now, and we can just exclude the method in Function (#6321)
* remove gemmlowp submodule (#6341)
* [NNAPI] Add pow support (#6310)
* Add support for running Android emulator from build.py on Windows. (#6317)
* fix the pipeline failure (#6346)
* Train BERT Using BFloat16 on A100 (#6090)
* traing bert using bf16
* Adam support bf16
* bugfix
* add fusedmatmul support
* fix after merge from master.
* bugfix
* bugfix after merge from master
* fast reduction for bf16.
* resolve comments
* fix win build
* bugfix
* change header file.
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
* Fix DerefNullPtr issues raised by SDLNativeRules. (#6348)
* update quantize to support basic optimization and e2e example for image classification (#6313)
update the resnet50-v1 to standard one from onnx zoo.
add an example for mobilenet
run basic optimization before quantization
fix a bug in Clip
* Enable graph save for orttrainer (#6333)
* Enable graph save for orttrainer
* Fix CI
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
* Update orttraining/orttraining/python/training/orttrainer_options.py
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
* Add PREfast to python packaging pipeline (#6343)
* Add PREfast to python packaging pipeline
* fix longformer benchmark io_binding output_buffers (#6345)
* fix longformer benchmark io_binding output_buffers
* format
* import benchmark_helper from parent directory.
* Use readelf for minimal build binary size checks. (#6338)
* Use readelf for minimal build binary size checks.
The on-disk size grows in 4KB chunks which makes it hard to see how much growth an individual checkin causes.
Only downside is that the sum of the sections is larger than the on-disk size (assumably things get packed smaller on disk and some of the section alignment constraints can be ignored)
* Remove unused function
* Java: Set C language warnings to W4 and adjust JNI code (#6347)
Set /W3 for C language and fix up JNI warnings.
* Pipeline Parallel Experimental Python API (#5815)
* Add create session to WinML telemetry to track WinML Usage (#6356)
* Fix one more SDL warning (#6359)
* fix -Wdangling-gsl (#6357)
* Add python example of TensorRT INT8 inference on ResNet model (#6255)
* add trt int8 example on resnet model
* Update e2e_tensorrt_resnet_example.py
* remove keras dependency and update class names
* move ImageNetDataReader and ImageClassificationEvaluator to tensorrt resnet example
* simplify e2e_tensorrt_resnet_example.py
* Update preprocessing.py
* merge tensorrt_calibrate
* Update calibrate.py
* Update calibrate.py
* generalize calibrate
* Update calibrate.py
* fix issues
* fix formating
* remove augment_all
* This added telemetry isn't needed (#6363)
* Wezuo/memory analysis (#5658)
* merged alloc_plan
* pass compilation
* Start running, incorrect allocation memory info
* add in comments
* fix a bug of recording pattern too early.
* debugging lifetime
* fix lifetime
* passed mnist
* in process of visualization
* Add code to generate chrome trace for allocations.
* in process of collecting fragmentation
* before rebuild
* passed mnist
* passed bert tiny
* fix the inplace reuse
* fix the exception of weight in pinned memory
* add guards to ensure the tensor is in AllocPlan
* add customized profiling
* debugging
* debugging
* fix the reuse of differnt location type
* add rank
* add the rank
* add fragmentation
* add time_step_trace
* Add summary for each execution step (total bytes, used/free bytes).
* add top k
* change type of top k parameter
* remove prints
* change heap to set{
* add the name pattern
* add the useage for pattern
* add partition
* change to static class
* add custom group
* remove const
* update memory_info
* in process of adding it as runtime config
* change the memory profiling to be an argument
* add some comments
* add checks to recored meomry_info in traaining session
* set the "local rank setting" to correct argument.
* addressing comments
* format adjustment
* formatting
* remove alloc_interval
* update memory_info.cc to skip session when there is no tensor for a particular memory type
* fix memory_info multiple iteration seg-fault
* consolidate mainz changes
* fixed some minor errors
* guard by ORT_MINIMAL_BUILD
* add ORT_MEMORY_PROFILE flag
* added compiler flag to turn on/off memory profiling related code
* clean up the code regarding comments
* add comments
* revoke the onnx version
* clean up the code to match master
* clean up the code to match master
* clean up the code to match master
Co-authored-by: Jesse Benson <benson.jesse@gmail.com>
Co-authored-by: Wei Zuo <wezuo@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-mgtbby.eastus.cloudapp.azure.com>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-yclzsf.eastus.cloudapp.azure.com>
* Support MLFloat16 in CumSum Cuda op for Opset 14 (#6355)
* Add CumSum-14 for Cuda
* fix convert_common version retrival (#6382)
* Refine auto_pad based pad computation in ConvTranspose (#6305)
* Fix SDL warning (#6390)
* Add max_norm for gradient clipping. (#6289)
* add max_norm as user option for gradient clipping
* add adam and lamb test cases for clip norm
* add frontend tests
* Add the custom op project information (#6334)
* Dont use default string marshalling in C# (#6219)
* Fix Windows x86 compiler warnings in the optimizers project (#6377)
* [Perf] Optimize Tile CPU and CUDA kernels for a corner case (#6376)
* Unblock Android CI code coverage failure (#6393)
* fix build on cuda11 (#6394)
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
* Load the model path correctly (#6369)
* Fix some compile warnings (#6316)
* OpenVino docker file changes to bypass privileged mode
Description: Builds and installs libusb without UDEV support, which is used for communicating with the VPU device.
Motivation and Context
This enables the resulting docker container to be run without '--privileged' and '--network host' options which may not be suitable in deployment environments.
* Megatron checkpointing (#6293)
* Add bart fairseq run script
* Add frontend change to enable megatron
* Initial changes for checkpointing
* Megatron optim state loading, checkpoint aggregation, frontend distributed tests for H, D+H
* Add load_checkpoint changes
* Fix CI
* Cleanup
* Fix CI
* review comments
* review comments
* review comments:
* Fix generate_submodule_cgmanifest.py Windows issues. (#6404)
* Continue memory planning when unknown shape tensor is encountered. (#6413)
* Reintroduce experimental api changes and fix remote build break (#6385)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* Add support for custom ops to minimal build. (#6228)
* Add support for custom ops to minimal build.
Cost is only ~8KB so including in base minimal build.
* enable pipeline to run quantization tests (#6416)
* enable pipeline to run quantization tests
setup test pipeline for quantization
* Minor cmake change (#6431)
* Liqun/liqun/enable pipeline parallel test2 (#6399)
* enable data and pipeline parallism test
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
* Farewell TrainableDropout (#5793)
* Deprecate TrainableDropout kernel.
* Update bert_toy_postprocessed.onnx to opset 12.
* Add more dropout tests.
* Fix BiasDropout kernel.
Co-authored-by: Ubuntu <OrtTrainingDev3@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sherlock Huang <bahuang@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sergii Dymchenko <sedymche@microsoft.com>
* fix null dereference warning (#6437)
* Expose graph ModelPath to TensorRT shared library (#6353)
* Update graph_viewer.cc
* Update tensorrt_execution_provider.cc
* Update graph_viewer.h
* Update tensorrt_execution_provider.cc
* Update tensorrt_execution_provider.cc
* Update provider_api.h
* Update provider_bridge_ort.cc
* Update provider_interfaces.h
* Update provider_interfaces.h
* expose GraphViewer ModelPath API to TRT shared lib
* add modelpath to compile
* update
* add model_path to onnx tensorrt parser
* use GenerateMetaDefId to generate unique TRT kernel name
* use GenerateMetaDefId to generate unique TRT engine name
* fix issue
* Update tensorrt_execution_provider.cc
* remove GetVecHash
* Update tensorrt_execution_provider.h
* convert wchar_t to char for tensorrt parser
* update tensorrt parser to include latest changes
* fix issues
* Update tensorrt_execution_provider.cc
* merge trt parser latest change
* add PROVIDER_DISALLOW_ALL(Path)
* add tool for generating test data for longformer (#6415)
* only build experimental api in redist (#6465)
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
* Add an option to save the training graph after optimization (#6410)
* expose optimized_model_filepath in SessionOptions as `debug.graph_save_paths.model_with_training_graph_after_optimization_path` in `ORTTrainerOptions`
* Share allocator between CUDA EP & TRT EP. (#6332)
* Share allocator between CUDA EP & TRT EP.
limitation:
1. Does not cover the per-thread allocator created by CUDA EP, still need to figure out the way to remove it
2. Need to have more identifiers to make it able to share CPU allocator across all EPs
* fix max norm clipping test in python packaging pipeline test (#6468)
* fix python packaging pipeline
* make clip norm test compatabile with both V100 and M60 GPUs
* Initial version of CoreML EP (#6392)
* Bug 31463811: Servicing: Redist (Nuget) conflicts with Microsoft.AI.MachineLearning starting 21H1+ (#6460)
* update load library code to have the fullly qualified path
* make it work for syswow32
* git Revert "make it work for syswow32"
This reverts commit b9f594341b7cf07241b18d0c376af905edcabae3.
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
* dequantize 1st input of lstm back if it is quantized (#6444)
* [java] Adds support for OrtEnvironment thread pools (#6406)
* Updates for Gradle 7.
* Adding support for OrtThreadingOptions into the Java API.
* Fixing a typo in the JNI code.
* Adding a test for the environment's thread pool.
* Fix cuda test, add comment to failure.
* Updating build.gradle
* fix SDL native rule warning #6246 (#6461)
* fix SDL rule (#6464)
* use tickcount64 (#6447)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* Update pypi package metadata (#6354)
* Update setup file data
* add missing comma
* remove python 3.5
* fix typo bracket
* Delete nuget extra configs (#6477)
* Op kernel type reduction infrastructure. (#6466)
Add infrastructure to support type reduction in Op kernel implementations.
Update Cast and IsInf CPU kernels to use it.
* Fixing a leak in OnnxSequences with String keys or values. (#6473)
* Increase the distributes tests pipeline timeout to 120 minutes (#6479)
* [CoreML EP] Add CI for CoreML EP (macOS) and add coreml_flags for EP options (#6481)
* Add macos coreml CI and coreml_flags
* Move save debuggubg model to use environment var
* Move pipeline off from macos CI template
* Fix an issue building using unix make, add parallel to build script
* Fixed build break for shared_lib and cmpile warning
* Fix a compile warning
* test
* Revert the accidental push from another branch
This reverts commit 472029ba25d50f9508474c9eeceb3454cead7877.
* Add ability to track per operator types in reduced build config. (#6428)
* Add ability to generate configuration that includes required types for individual operators, to allow build size reduction based on that.
- Add python bindings for ORT format models
- Add script to update bindings and help info
- Add parsing of ORT format models
- Add ability to enable type reduction to config generation
- Update build.py to only allow operator/type reduction via config
- simpler to require config to be generated first
- can't mix a type aware (ORT format model only) and non-type aware config as that may result in insufficient types being enabled
- Add script to create reduced build config
- Update CIs
* merge e2e with distributed pipeline (#6443)
merge e2e with distributed pipeline
* Fix test breaks in Windows ingestion pipeline (#6476)
* fix various build breaks with Windows build
* fix runtime errors loading libraries from system32
* add build_inbox check to winml_test_common
* use raw string
* cleanup
* fix dll load
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
* Speed up the Mac CI runs (#6483)
* expose learningmodelpixelrange property (#5877)
* Fix of support api version bug for [de]quantize (#6492)
* SDL fixes: add proper casts/format specifiers (#6446)
* SDL annotation fixes (#6448)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* [OpenVINO-EP] Remove support for OpenVINO 2020.2 (#6493)
* Removed OpenVINO 2020.2 support
* Updated documentation and build.py
* Removed unnecessary libraries from setup.py
* Support pad operator in quantization and quantized nhwc transformer. Fix Pad operator bug. (#6325)
Support pad operator in quantization tool.
Support pad operator in quantized nhwc transformer.
Fix pad() operator bug when pad input's inner(right) most axis value is zero for Edge and Reflect mode, it copied wrong value to the cells to be padded. Note the Constant mode will not trigger this bug, as Edge/Reflect need copy value from the already copied array while Constant mode only fill specified value.
Add more test cases to cover pad() operator bug fixed here.
Fix quantization tools uint8/int8 value overflow issue when quantize weights in python.
* Improve work distribution for Expand operator, and sharded LoopCounter configuration (#6454)
Description: This PR makes two changes identified while looking at a PGAN model.
First, it uses ThreadPool::TryParallelFor for the main parallel loops in the Expand operator. This lets the thread pool decide on the granularity at which to distribute work (unlike TrySimpleParallelFor). Profiling showed high costs when running "simple" loops with 4M iterations each of which copied only 4 bytes.
Second, it updates the sharded loop counter in the thread pool so that the number of shards is capped by the number of threads. This helps make the performance of any other high-contention "simple" loops more robust at low thread counts by letting each thread work on its own "home" shard for longer.
Motivation and Context
Profiling showed a PGAN model taking 2x+ longer with the non-OpenMP build. The root cause was that the OpenMP build uses simple static scheduling of loop iterations, while the non-OpenMP build uses dynamic scheduling. The combination of large numbers of tiny iterations is less significant with static scheduling --- although still desirable to avoid, given that each iteration incurs a std::function invocation.
* Update document of transformer optimization (#6487)
* nuphar test to avoid test data download to improve passing rate (#6467)
nuphar test to avoid test data download to improve passing rate
* Fuse cuda conv with activation (#6351)
* optimize cuda conv by fused activation
* remove needless print out
* exclude test from cpu
* handle status error from cudnn 8.x
* add reference to base class
* add hipify
* [CoreML EP] Add support for some activations/Transpose, move some shared helpers from NNAPI to shared space (#6498)
* Init change
* Move some helper from nnapi ep to shared
* Add transpose support
* Fix trt ci build break
* Refine transformers profiler output (#6502)
* output nodes in the original order; grouped by node name
* add document for profiler
* Update to match new test setup. (#6496)
* Update to match new test setup.
* Add Gemm(7) manually for now.
Will fix properly on Monday. It's used by mnist.ort as that is created by optimizing mnist.onnx to level 1 causing 2 nodes to be replaced by a Gemm and the op to be missing from the required list as that is created using the original onnx model.
* Enable dense sequence optimized version of Pytorch exported BERT-L on AMD GPU (#6504)
* Permit dense seq optimization on BERT-L pytorch export by enabling ReduceSumTraining, Equal, and NonZero on AMD
* enable Equal tests
* enable fast_matrix_reduction test case
* Optimize GatherGrad for AMD GPU (#6381)
* optimize gathergrad
* address comments
Co-authored-by: Weixing Zhang <wezhan@microsoft.com>
* add explicit barriers for buffer overread and overrwrite (#6484)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* fix sdl bugs for uninitialized variables and returns (#6450)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* handle hr error conditions (#6449)
Co-authored-by: Ori Levari <orlevari@microsoft.com>
* Dnnl training (#6045)
* Add ReluGrad and ConvGrad ops for the dnnl provider
* the mnist sample is updated to add the --use_dnnl option that
will cause the sample to use the dnnl execution provider for
nodes that exist in dnnl provider.
* Added the ability to find forward ops. Dnnl backward gradient
ops require the forward primitive description and workspace
from the forward operation.
* Enable specifying the execution provider for Gradient Checker Tests
* Prevent memory leak when running dnnl_provider in training mode
Prevent creating a SubgraphPrimitivePool when the code is built with the
ENABLE_TRAINING build flag. Instead create a SubgraphPrimitive directly.
The SubgraphPrimitivePool was causing a pool of SubgraphPrimitives to be
stashed in a map for reuse. Due to the way the Training Loop uses threads
the pool of SubgraphPrimitives were not being reuse instead a new pool of
SubgraphPrimitives being created each run. The old pool was not instantly
freed. This behavior could be a language error when using thread_local
memory.
Signed-off-by: George Nash <george.nash@intel.com>
* Added fixes to maxpoolgrad and memory leak.
Maxpoolgrad will now pass all unit tests.
With the conv and convgrad disabled for dnnl, mnist is able to train till 95%
Signed-off-by: Chethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
* Fixed misc issues when testing training code with dnnl provider
* fix conv_grad dnnl tests with dilation to run dnnl execution provider
* update mnist training sample to accept convolution type models
convolution models require the input shape to be {1, 28, 28}
instead of the flat {728} image that is used for the gemm models
this will enable models that require the different shape by adding
`--model_type conv` to the command line when running the mnist sample.
(while testing a workaround was used see #4762)
* Disable weight caching in dnnl conv operator when using training
When training we can not use cached weights because the weight
will be updated each run. This re-enables dnnl Conv and ConvGrad Ops.
The weight caching was the source of the error from Conv when training.
* Fix issues found when building grad ops on Linux
* The dnnl_convgrad code was over using the scope operator
causing a compilation problem.
* The dnnl_maxpoolgrad code had a logic error that is was
comparing with the source description when it should have
been comparing with the destination despription.
* Update BUILD.md so it shows DNNL for training
* Updated the table of contents. Since the same providers
are listed twice. Once for Infrance and again for Training
an HTML anchor was added to distinguish the second header
from the first for the TOC.
* Fix build failure when not using --enable-training build option
* reorganize the gradient operators so they are grouped together
* Fix issues found when running onnx_backend_test_series.py
* Pooling code only supports 2 outputs when built with --enable-training
* Address code review feedback
* class member variables end in underscore_
* use dst instead of dist to match pattern use elsewhere in DNNL code.
* Remove workaround that was introduced to handle problems running
convolution based training models. See issue #4762
Signed-off-by: George Nash <george.nash@intel.com>
* Isolate training code and code cleanup
* Do not build if dnnl_gpu_runtime if enable_training is set training code
does not support dnnl_gpu_runtime yet.
* Isolated Training code inside ifdefs so that they wont affect
project if built without training enabled
* Inadvertant changes in whitespace were removed to make code review simpler
* Undid some code reordering that was not needed
* comments added to closing #endif statments to simplify reading complex ifdefs
* Modified the GetPrimitiveDesc functions to return shared_ptr instead of raw
pointer. This matches what was done in Pool code and is safer memory code.
Signed-off-by: George Nash <george.nash@intel.com>
* Address code review issues
- whitespace changes caused by running clang-format on the code
- Several spelling errors fixed
- Removed/changed some ifdefs to improve readability
- other misc. changes in responce to code review.
Signed-off-by: George Nash <george.nash@intel.com>
* Code changes to address code review
- Simplify iteration code using `auto` keyword
- remove C style cast that was not needed
- remove instance variable that was not needed [relugrad.h]
- added the execution providers to `ComputeGradientErrorInternal()`
and `ComputeTheoreticalJacobianTranspose()` instead of using
a pointer to an instance varaible [gradient_checker.h/.cc]
Signed-off-by: George Nash <george.nash@intel.com>
* Combined the default gradient ops test and dnnl gradient ops test for ConvGrad and MaxPoolGrad into one function with the help of a helper function.
This will reduce repeated code.
Signed-off-by: Palangotu Keshava, Chethan's avatarChethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
* Replaced the stack used by convgrad to vector so that the vector(used as stack) can be easily cleared everytime the graph is created.
This will prevent memory leak from convolution kernels being pushed constantly onto the stack.
Signed-off-by: chethan.palangotu.keshava@intel.com
* Code clean up and formating updates
- Removed empty else statment
- updated indentation of code that was causing double curly brackets to look unususal
- Changed check for NumDimensions to Size in Relu and ReluGrad error checking code.
- isolated training code
Signed-off-by: George Nash <george.nash@intel.com>
* Restore inadvertantly removed ConvGrad tests
When combining the DNNL and CPU version of the ConvGrad
tests two test were inadvertantly excluded. This adds
back the Conv3d and Conv3d with strides test cases.
Signed-off-by: George Nash <george.nash@intel.com>
* Add validation to ConvGrad
This validates the dimensions of the ConvGrad match the
passed in Convolution forward primitive description.
The current code for DNNL ConvGrad makes the assumption that the ConvGrad
nodes will be visited in the reverse order from the corresponding Conv nodes
The added validation will return an error if this assumption is not true.
Signed-off-by: George Nash <george.nash@intel.com>
* Do not create new execution providers in provider_test_utils
This removes the code that generated new execution providers in the
OpTester::Run function. This was added because the std::move was
leaving the `entry` value empty so subsequent calls would cause a
segfault.
Problem is this potentially changed the execution_provider because it
would create the default provider dropping any custom arguments.
When the now removed code was originally added the std::move was causing
crashes when the GradientChecker unit tests were run. However, it is no
longer causing problems even with the code removed.
Signed-off-by: George Nash <george.nash@intel.com>
* Change the forward conv stack to a forward conv map
This changes how the forward conv kernel is mapped to the bwd ConvGrad
kernel the problematic stack is no longer used.
The convolution stack made the assumption that the corresponding
ConvGrad operator would be visited in reverse order of the forward
Conv operators. This was always problematic and was unlikely to
work for inception models.
Important changes:
- The weight_name is added to the ConvGrad dnnl_node making it
possible to use the weight_name as a lookup key to find the
Conv forward Kernel
- the `std::vector fwd_conv_stack_` has been replaced with a
`std::map fwd_conv_kernel_map_`
- Although it is not needed lock_guards were added when writing
to and reading from the fwd_conv_kernel_map_ as well as the
fwd_kernel_map_. These should always be accessed by a single
thread when preparing the dnnl subgraphs so the guard should not
be needed but its added just in case.
- Updated the comments ConvGrad.h code to no longer mention the
stack. The error check is not removed. It will be good to verify
there are no errors as we continue to test against more models.
Signed-off-by: George Nash <george.nash@intel.com>
Co-authored-by: Chethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
Co-authored-by: unknown <63478620+jeyblu@users.noreply.github.com>
* Lochi/refactor yolov3 quantization (#6290)
* Refactor the code and move data reader, preprocessing, evaluation to
E2E_example_mode
* Refactor the code.
Move data reader, preprocessing, evaluation to model specific example
under E2E_example_mode
* refactor code
* Move yolov3 example to specific folder and add additional pre/post
processing
* Print a warning message for using newer c_api header on old binary (#6507)
* Fix issues with ArmNN build setup (#6495)
* ArmNN build fixes
* Update BUILD.md to document that the ACL paths must be specified to build ArmNN
* Fix CUDA build error. We don't setup the link libraries correctly/consistently so improve that.
* Fix Windows CI builds by updating test scripts to work with numpy 1.20. (#6518)
* Update onnxruntime_test_python.py to work with numpy 1.20.
Some aliases are deprecated in favor of the built-in python types. See https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
np.array with bytes for entries and dtype of np.void no longer automatically pads. Change a test to adjust for that.
* Fix another test script
* Fix ORTModule branch for orttraining-* pipelines
* Update pytorch nightly version dependency
Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com>
Co-authored-by: George Wu <jywu@microsoft.com>
Co-authored-by: Cecilia Liu <ziyue.liu7@gmail.com>
Co-authored-by: Ryan Hill <38674843+RyanUnderhill@users.noreply.github.com>
Co-authored-by: George Nash <george.nash@intel.com>
Co-authored-by: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com>
Co-authored-by: Yateng Hong <toothache9010@gmail.com>
Co-authored-by: stevenlix <38092805+stevenlix@users.noreply.github.com>
Co-authored-by: Derek Murray <Derek.Murray@microsoft.com>
Co-authored-by: ashbhandare <ash.bhandare@gmail.com>
Co-authored-by: Scott McKay <skottmckay@gmail.com>
Co-authored-by: Changming Sun <chasun@microsoft.com>
Co-authored-by: Tracy Sharpe <42477615+tracysh@users.noreply.github.com>
Co-authored-by: Juliana Franco <jufranc@microsoft.com>
Co-authored-by: Pranav Sharma <prs@microsoft.com>
Co-authored-by: Tixxx <tix@microsoft.com>
Co-authored-by: Jay Rodge <jayrodge@live.com>
Co-authored-by: Du Li <duli1@microsoft.com>
Co-authored-by: Du Li <duli@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Yufeng Li <liyufeng1987@gmail.com>
Co-authored-by: baijumeswani <bmeswani@microsoft.com>
Co-authored-by: Sergii Dymchenko <sedymche@microsoft.com>
Co-authored-by: jingyanwangms <47403504+jingyanwangms@users.noreply.github.com>
Co-authored-by: satyajandhyala <satya.k.jandhyala@gmail.com>
Co-authored-by: S. Manohar Karlapalem <manohar.karlapalem@intel.com>
Co-authored-by: Weixing Zhang <weixingzhang@users.noreply.github.com>
Co-authored-by: Suffian Khan <sukha@microsoft.com>
Co-authored-by: Olivia Jain <oljain@microsoft.com>
Co-authored-by: Chi Lo <54722500+chilo-ms@users.noreply.github.com>
Co-authored-by: Hariharan Seshadri <shariharan91@gmail.com>
Co-authored-by: Ryan Lai <rylai@microsoft.com>
Co-authored-by: Jesse Benson <jesseb@microsoft.com>
Co-authored-by: sfatimar <64512376+sfatimar@users.noreply.github.com>
Co-authored-by: suryasidd <surya.siddharth.pemmaraju@intel.com>
Co-authored-by: sfatimar <sahar.fatima@intel/com>
Co-authored-by: MaajidKhan <n.maajidkhan@gmail.com>
Co-authored-by: mohdansx <mohdx.ansari@intel.com>
Co-authored-by: Xavier Dupré <xadupre@users.noreply.github.com>
Co-authored-by: Michael Goin <mgoin@vols.utk.edu>
Co-authored-by: Michael Giba <michaelgiba@gmail.com>
Co-authored-by: William Tambellini <wtambellini@sdl.com>
Co-authored-by: Hector Li <hecli@microsoft.com>
Co-authored-by: Aishwarya <aibhanda@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: liqunfu <liqfu@microsoft.com>
Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: pengwa <pengwa@microsoft.com>
Co-authored-by: Tang, Cheng <souptc@gmail.com>
Co-authored-by: Cheng Tang <chenta@microsoft.com>
Co-authored-by: Tianlei Wu <tlwu@microsoft.com>
Co-authored-by: Ye Wang <52801275+wangyems@users.noreply.github.com>
Co-authored-by: Chun-Wei Chen <jacky82226@gmail.com>
Co-authored-by: Vincent Wang <wangwchpku@outlook.com>
Co-authored-by: Vincent Wang <weicwang@microsoft.com>
Co-authored-by: Luyao Ren <375833274@qq.com>
Co-authored-by: Zhang Lei <zhang.huanning@hotmail.com>
Co-authored-by: Tim Harris <tiharr@microsoft.com>
Co-authored-by: Ashwini Khade <askhade@microsoft.com>
Co-authored-by: Dmitri Smirnov <yuslepukhin@users.noreply.github.com>
Co-authored-by: Alberto Magni <49027342+alberto-magni@users.noreply.github.com>
Co-authored-by: Wei-Sheng Chin <wschin@outlook.com>
Co-authored-by: wezuo <49965641+wezuo@users.noreply.github.com>
Co-authored-by: Jesse Benson <benson.jesse@gmail.com>
Co-authored-by: Wei Zuo <wezuo@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-mgtbby.eastus.cloudapp.azure.com>
Co-authored-by: wezuo <wezuo@az-eus-v100-32gb-5-worker-yclzsf.eastus.cloudapp.azure.com>
Co-authored-by: Wenbing Li <10278425+wenbingl@users.noreply.github.com>
Co-authored-by: Martin Man <supermt@gmail.com>
Co-authored-by: M. Zeeshan Siddiqui <mzs@microsoft.com>
Co-authored-by: Ori Levari <ori.levari@microsoft.com>
Co-authored-by: Ori Levari <orlevari@microsoft.com>
Co-authored-by: Ubuntu <OrtTrainingDev3@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sherlock Huang <bahuang@OrtTrainingDev3.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Sheil Kumar <smk2007@gmail.com>
Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
Co-authored-by: Ryota Tomioka <ryoto@microsoft.com>
Co-authored-by: Adam Pocock <adam.pocock@oracle.com>
Co-authored-by: Yulong Wang <f.s@qq.com>
Co-authored-by: Faith Xu <faxu@microsoft.com>
Co-authored-by: Xiang Zhang <xianz@microsoft.com>
Co-authored-by: suryasidd <48925384+suryasidd@users.noreply.github.com>
Co-authored-by: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com>
Co-authored-by: Weixing Zhang <wezhan@microsoft.com>
Co-authored-by: Chethan Palangotu Keshava <chethan.palangotu.keshava@intel.com>
Co-authored-by: unknown <63478620+jeyblu@users.noreply.github.com>
2021-02-02 16:59:56 +00:00
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ t e n s o r r t >
2020-08-13 22:24:44 +00:00
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
2020-08-13 22:24:44 +00:00
)
endif ( )
2022-01-10 23:18:43 +00:00
if ( onnxruntime_USE_MIGRAPHX )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ m i g r a p h x >
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2019-11-06 03:55:46 +00:00
if ( onnxruntime_USE_OPENVINO )
2020-04-24 11:06:02 +00:00
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
2021-02-26 14:34:43 +00:00
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ o p e n v i n o >
2020-11-21 01:39:57 +00:00
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
2020-04-24 11:06:02 +00:00
)
2019-11-06 03:55:46 +00:00
endif ( )
2022-06-17 21:49:04 +00:00
if ( DEFINED ENV{OPENVINO_MANYLINUX} )
file ( GLOB onnxruntime_python_openvino_python_srcs CONFIGURE_DEPENDS
" $ { O N N X R U N T I M E _ R O O T } / c o r e / p r o v i d e r s / o p e n v i n o / s c r i p t s / * "
)
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { o n n x r u n t i m e _ p y t h o n _ o p e n v i n o _ p y t h o n _ s r c s }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2021-05-20 14:53:47 +00:00
if ( onnxruntime_USE_CUDA )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ c u d a >
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2022-09-22 21:53:40 +00:00
if ( onnxruntime_USE_CANN )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ c a n n >
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2021-11-02 02:12:09 +00:00
if ( onnxruntime_USE_ROCM )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ r o c m >
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2020-09-08 21:34:09 +00:00
if ( onnxruntime_USE_DML )
2023-05-25 00:20:40 +00:00
if ( NOT onnxruntime_USE_CUSTOM_DIRECTML )
2023-05-01 19:02:56 +00:00
set ( dml_shared_lib_path ${ DML_PACKAGE_DIR } /bin/ ${ onnxruntime_target_platform } -win/ ${ DML_SHARED_LIB } )
else ( )
set ( dml_shared_lib_path ${ DML_PACKAGE_DIR } /bin/ ${ DML_SHARED_LIB } )
endif ( )
2020-09-08 21:34:09 +00:00
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
2023-05-01 19:02:56 +00:00
$ { d m l _ s h a r e d _ l i b _ p a t h }
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
2020-09-08 21:34:09 +00:00
)
endif ( )
2020-11-20 23:18:35 +00:00
if ( onnxruntime_USE_NNAPI_BUILTIN )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ n n a p i >
2021-02-24 04:21:57 +00:00
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
2020-11-20 23:18:35 +00:00
)
endif ( )
2021-07-29 17:06:47 +00:00
if ( onnxruntime_USE_COREML )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ c o r e m l >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2021-08-28 18:05:21 +00:00
2024-04-16 05:33:12 +00:00
if ( onnxruntime_USE_QNN )
2025-01-22 20:11:00 +00:00
if ( NOT onnxruntime_BUILD_QNN_EP_STATIC_LIB )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ q n n >
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ s h a r e d >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2024-04-16 05:33:12 +00:00
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ { Q N N _ L I B _ F I L E S }
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
2024-09-19 06:24:32 +00:00
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ q n n _ c t x _ g e n >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
2024-04-29 16:44:54 +00:00
if ( EXISTS "${onnxruntime_QNN_HOME}/Qualcomm AI Hub Proprietary License.pdf" )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
" $ { o n n x r u n t i m e _ Q N N _ H O M E } / Q u a l c o m m A I H u b P r o p r i e t a r y L i c e n s e . p d f "
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e /
)
endif ( )
2024-04-16 05:33:12 +00:00
endif ( )
2024-12-02 21:57:30 +00:00
if ( onnxruntime_USE_VSINPU )
add_custom_command (
T A R G E T o n n x r u n t i m e _ p y b i n d 1 1 _ s t a t e P O S T _ B U I L D
C O M M A N D $ { C M A K E _ C O M M A N D } - E c o p y
$ < T A R G E T _ F I L E : o n n x r u n t i m e _ p r o v i d e r s _ v s i n p u >
$ < T A R G E T _ F I L E _ D I R : $ { b u i l d _ o u t p u t _ t a r g e t } > / o n n x r u n t i m e / c a p i /
)
endif ( )
2021-04-29 18:54:57 +00:00
endif ( )
