mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
Merge remote-tracking branch 'upstream/master' into user/justoeck/ri_20210525
This commit is contained in:
Justin Stoecker
commit
3fb4a2dc99
392 changed files with 6603 additions and 4314 deletions
|
|
@ -151,7 +151,7 @@ option(onnxruntime_USE_MPI "Build with MPI support" OFF)
|
|||
option(onnxruntime_BUILD_WEBASSEMBLY "Enable this option to create WebAssembly byte codes" OFF)
|
||||
option(onnxruntime_ENABLE_WEBASSEMBLY_THREADS "Enable this option to create WebAssembly byte codes with multi-threads support" OFF)
|
||||
option(onnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING "Enable this option to turn on exception catching" OFF)
|
||||
option(onnxruntime_ENABLE_WEBASSEMBLY_SOURCEMAP "Enable this option to turn on sourcemap" OFF)
|
||||
option(onnxruntime_ENABLE_WEBASSEMBLY_DEBUG_INFO "Enable this option to turn on DWARF format debug info" OFF)
|
||||
|
||||
# Enable bitcode for iOS
|
||||
option(onnxruntime_ENABLE_BITCODE "Enable bitcode for iOS only" OFF)
|
||||
|
|
@ -194,7 +194,9 @@ if(onnxruntime_USE_VALGRIND AND NOT WIN32)
|
|||
endif()
|
||||
|
||||
if (onnxruntime_ENABLE_NVTX_PROFILE)
|
||||
add_definitions(-DENABLE_NVTX_PROFILE=1)
|
||||
message(WARNING "NTVX profile temporarily disabled, will be fixed soon")
|
||||
# TODO: This doesn't work with the shared cuda provider. Disabling temporarily to do a clean fix later as it wasn't trivial
|
||||
# add_definitions(-DENABLE_NVTX_PROFILE=1)
|
||||
endif()
|
||||
|
||||
if (onnxruntime_ENABLE_BITCODE)
|
||||
|
|
@ -283,12 +285,13 @@ if (onnxruntime_BUILD_WEBASSEMBLY)
|
|||
string(APPEND CMAKE_CXX_FLAGS " -flto")
|
||||
endif()
|
||||
|
||||
if (onnxruntime_ENABLE_WEBASSEMBLY_SOURCEMAP)
|
||||
# Build with sourcemap support
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-g4 --source-map-base ${onnxruntime_WEBASSEMBLY_SOURCEMAP_BASE}")
|
||||
if (onnxruntime_ENABLE_WEBASSEMBLY_DEBUG_INFO)
|
||||
# "-g3" generates DWARF format debug info.
|
||||
# NOTE: With debug info enabled, web assembly artifacts will be very huge (>1GB). So we offer an option to build without debug info.
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-g3")
|
||||
else()
|
||||
# override default "-g3" as it generates super huge (1GB+) WASM targets which fails to load
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-g2")
|
||||
# do not generate any debug info. This helps to accelerate building process and reduce binary size.
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-g0")
|
||||
endif()
|
||||
|
||||
# Build WebAssembly with multi-threads support.
|
||||
|
|
@ -1399,7 +1402,6 @@ if (onnxruntime_USE_CUDA)
|
|||
list(APPEND ONNXRUNTIME_CUDA_LIBRARIES cublas cudnn curand cufft)
|
||||
endif()
|
||||
|
||||
list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${ONNXRUNTIME_CUDA_LIBRARIES})
|
||||
if(NOT CMAKE_CUDA_ARCHITECTURES)
|
||||
if(CMAKE_LIBRARY_ARCHITECTURE STREQUAL "aarch64-linux-gnu")
|
||||
# Support for Jetson/Tegra ARM devices
|
||||
|
|
|
|||
|
|
@ -135,7 +135,6 @@ target_link_libraries(onnxruntime PRIVATE
|
|||
${PROVIDERS_ACL}
|
||||
${PROVIDERS_ARMNN}
|
||||
${PROVIDERS_COREML}
|
||||
${PROVIDERS_CUDA}
|
||||
${PROVIDERS_DML}
|
||||
${PROVIDERS_MIGRAPHX}
|
||||
${PROVIDERS_NNAPI}
|
||||
|
|
|
|||
|
|
@ -90,6 +90,11 @@ source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_common_src})
|
|||
|
||||
onnxruntime_add_static_library(onnxruntime_common ${onnxruntime_common_src})
|
||||
|
||||
if (onnxruntime_USE_CUDA)
|
||||
# Some files, like the provider_bridge_ort include files that depend on cuda headers
|
||||
target_include_directories(onnxruntime_common PUBLIC ${onnxruntime_CUDA_HOME}/include)
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_TELEMETRY)
|
||||
set_target_properties(onnxruntime_common PROPERTIES COMPILE_FLAGS "/FI${ONNXRUNTIME_INCLUDE_DIR}/core/platform/windows/TraceLoggingConfigPrivate.h")
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -33,9 +33,16 @@ if(onnxruntime_ENABLE_INSTRUMENT)
|
|||
endif()
|
||||
if(onnxruntime_USE_TENSORRT)
|
||||
# TODO: for now, core framework depends on CUDA. It should be moved to TensorRT EP
|
||||
target_include_directories(onnxruntime_framework PRIVATE ${ONNXRUNTIME_ROOT} ${onnxruntime_CUDNN_HOME}/include PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
target_include_directories(onnxruntime_framework PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${onnxruntime_CUDNN_HOME}/include PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
else()
|
||||
target_include_directories(onnxruntime_framework PRIVATE ${ONNXRUNTIME_ROOT} PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
|
||||
target_include_directories(onnxruntime_framework PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
|
||||
endif()
|
||||
# Needed for the provider interface, as it includes training headers when training is enabled
|
||||
if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
|
||||
target_include_directories(onnxruntime_framework PRIVATE ${ORTTRAINING_ROOT})
|
||||
if (onnxruntime_USE_NCCL OR onnxruntime_USE_MPI)
|
||||
target_include_directories(onnxruntime_framework PUBLIC ${MPI_CXX_INCLUDE_DIRS})
|
||||
endif()
|
||||
endif()
|
||||
onnxruntime_add_include_to_target(onnxruntime_framework onnxruntime_common onnx onnx_proto protobuf::libprotobuf flatbuffers)
|
||||
set_target_properties(onnxruntime_framework PROPERTIES FOLDER "ONNXRuntime")
|
||||
|
|
|
|||
|
|
@ -60,6 +60,12 @@ file(GLOB_RECURSE onnxruntime_ir_defs_src CONFIGURE_DEPENDS
|
|||
"${ONNXRUNTIME_ROOT}/core/defs/*.cc"
|
||||
)
|
||||
|
||||
if (onnxruntime_ENABLE_TRAINING_OPS AND NOT onnxruntime_ENABLE_TRAINING)
|
||||
set(orttraining_graph_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/core/graph/training_op_defs.cc"
|
||||
"${ORTTRAINING_SOURCE_DIR}/core/graph/training_op_defs.h"
|
||||
)
|
||||
endif()
|
||||
if (onnxruntime_ENABLE_TRAINING)
|
||||
file(GLOB_RECURSE orttraining_graph_src CONFIGURE_DEPENDS
|
||||
"${ORTTRAINING_SOURCE_DIR}/core/graph/*.h"
|
||||
|
|
@ -68,7 +74,7 @@ if (onnxruntime_ENABLE_TRAINING)
|
|||
endif()
|
||||
|
||||
set(onnxruntime_graph_lib_src ${onnxruntime_graph_src} ${onnxruntime_ir_defs_src})
|
||||
if (onnxruntime_ENABLE_TRAINING)
|
||||
if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
|
||||
list(APPEND onnxruntime_graph_lib_src ${orttraining_graph_src})
|
||||
endif()
|
||||
|
||||
|
|
@ -83,7 +89,7 @@ endif()
|
|||
|
||||
target_include_directories(onnxruntime_graph PRIVATE ${ONNXRUNTIME_ROOT})
|
||||
|
||||
if (onnxruntime_ENABLE_TRAINING)
|
||||
if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
|
||||
target_include_directories(onnxruntime_graph PRIVATE ${ORTTRAINING_ROOT})
|
||||
|
||||
if (onnxruntime_USE_NCCL)
|
||||
|
|
@ -95,7 +101,7 @@ set_target_properties(onnxruntime_graph PROPERTIES FOLDER "ONNXRuntime")
|
|||
set_target_properties(onnxruntime_graph PROPERTIES LINKER_LANGUAGE CXX)
|
||||
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/graph DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core)
|
||||
source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_graph_src} ${onnxruntime_ir_defs_src})
|
||||
if (onnxruntime_ENABLE_TRAINING)
|
||||
if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
|
||||
source_group(TREE ${ORTTRAINING_ROOT} FILES ${orttraining_graph_src})
|
||||
endif()
|
||||
|
||||
|
|
|
|||
|
|
@ -134,10 +134,18 @@ if (WIN32)
|
|||
if(NOT onnxruntime_ENABLE_STATIC_ANALYSIS)
|
||||
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:onnxruntime> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_FILE_NAME:onnxruntime>)
|
||||
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:onnxruntime4j_jni> ${JAVA_PACKAGE_JNI_DIR}/$<TARGET_FILE_NAME:onnxruntime4j_jni>)
|
||||
if (onnxruntime_USE_CUDA)
|
||||
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:onnxruntime_providers_shared> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_FILE_NAME:onnxruntime_providers_shared>)
|
||||
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:onnxruntime_providers_cuda> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_FILE_NAME:onnxruntime_providers_cuda>)
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:onnxruntime> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime>)
|
||||
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:onnxruntime4j_jni> ${JAVA_PACKAGE_JNI_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime4j_jni>)
|
||||
if (onnxruntime_USE_CUDA)
|
||||
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:onnxruntime_providers_shared> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime_providers_shared>)
|
||||
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:onnxruntime_providers_cuda> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime_providers_cuda>)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# run the build process (this copies the results back into CMAKE_CURRENT_BINARY_DIR)
|
||||
|
|
|
|||
|
|
@ -226,13 +226,18 @@ if (onnxruntime_USE_CUDA)
|
|||
"${ONNXRUNTIME_ROOT}/core/providers/cuda/*.h"
|
||||
"${ONNXRUNTIME_ROOT}/core/providers/cuda/*.cc"
|
||||
)
|
||||
# The shared_library files are in a separate list since they use precompiled headers, and the above files have them disabled.
|
||||
file(GLOB_RECURSE onnxruntime_providers_cuda_shared_srcs CONFIGURE_DEPENDS
|
||||
"${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.h"
|
||||
"${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.cc"
|
||||
)
|
||||
file(GLOB_RECURSE onnxruntime_providers_cuda_cu_srcs CONFIGURE_DEPENDS
|
||||
"${ONNXRUNTIME_ROOT}/core/providers/cuda/*.cu"
|
||||
"${ONNXRUNTIME_ROOT}/core/providers/cuda/*.cuh"
|
||||
)
|
||||
|
||||
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_cu_srcs})
|
||||
set(onnxruntime_providers_cuda_src ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_cu_srcs})
|
||||
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_shared_srcs} ${onnxruntime_providers_cuda_cu_srcs})
|
||||
set(onnxruntime_providers_cuda_src ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_shared_srcs} ${onnxruntime_providers_cuda_cu_srcs})
|
||||
|
||||
# disable contrib ops conditionally
|
||||
if(NOT onnxruntime_DISABLE_CONTRIB_OPS)
|
||||
|
|
@ -275,7 +280,7 @@ if (onnxruntime_USE_CUDA)
|
|||
list(APPEND onnxruntime_providers_cuda_src ${onnxruntime_cuda_training_ops_cc_srcs} ${onnxruntime_cuda_training_ops_cu_srcs})
|
||||
endif()
|
||||
|
||||
onnxruntime_add_static_library(onnxruntime_providers_cuda ${onnxruntime_providers_cuda_src})
|
||||
onnxruntime_add_shared_library_module(onnxruntime_providers_cuda ${onnxruntime_providers_cuda_src})
|
||||
|
||||
#target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:-Xcompiler \"/analyze:stacksize 131072\">")
|
||||
if (HAS_GUARD_CF)
|
||||
|
|
@ -302,8 +307,11 @@ if (onnxruntime_USE_CUDA)
|
|||
onnxruntime_add_include_to_target(onnxruntime_providers_cuda onnxruntime_training)
|
||||
target_link_libraries(onnxruntime_providers_cuda PRIVATE onnxruntime_training)
|
||||
endif()
|
||||
add_dependencies(onnxruntime_providers_cuda ${onnxruntime_EXTERNAL_DEPENDENCIES} ${onnxruntime_tvm_dependencies})
|
||||
target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
|
||||
add_dependencies(onnxruntime_providers_cuda onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES} ${onnxruntime_tvm_dependencies})
|
||||
target_link_libraries(onnxruntime_providers_cuda PRIVATE cudart cublas cudnn curand cufft onnxruntime_providers_shared)
|
||||
target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
# ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" inside tensor_shape.h is found
|
||||
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cuda DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
|
||||
set_target_properties(onnxruntime_providers_cuda PROPERTIES LINKER_LANGUAGE CUDA)
|
||||
set_target_properties(onnxruntime_providers_cuda PROPERTIES FOLDER "ONNXRuntime")
|
||||
|
|
@ -314,9 +322,13 @@ if (onnxruntime_USE_CUDA)
|
|||
|
||||
if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
|
||||
target_include_directories(onnxruntime_providers_cuda PRIVATE ${ORTTRAINING_ROOT} ${MPI_CXX_INCLUDE_DIRS})
|
||||
if(onnxruntime_USE_MPI)
|
||||
target_link_libraries(onnxruntime_providers_cuda PRIVATE ${MPI_LIBRARIES} ${MPI_CXX_LINK_FLAGS})
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_NCCL)
|
||||
target_include_directories(onnxruntime_providers_cuda PRIVATE ${NCCL_INCLUDE_DIRS})
|
||||
target_link_libraries(onnxruntime_providers_cuda PRIVATE ${NCCL_LIBRARIES})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
|
@ -349,6 +361,24 @@ if (onnxruntime_USE_CUDA)
|
|||
set_target_properties(onnxruntime_providers_cuda PROPERTIES
|
||||
STATIC_LIBRARY_FLAGS "${onnxruntime_providers_cuda_static_library_flags}")
|
||||
endif()
|
||||
|
||||
if(APPLE)
|
||||
set_property(TARGET onnxruntime_providers_cuda APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/core/providers/cuda/exported_symbols.lst")
|
||||
target_link_libraries(onnxruntime_providers_cuda PRIVATE nsync_cpp)
|
||||
elseif(UNIX)
|
||||
set_property(TARGET onnxruntime_providers_cuda APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/core/providers/cuda/version_script.lds -Xlinker --gc-sections")
|
||||
target_link_libraries(onnxruntime_providers_cuda PRIVATE nsync_cpp)
|
||||
elseif(WIN32)
|
||||
set_property(TARGET onnxruntime_providers_cuda APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${ONNXRUNTIME_ROOT}/core/providers/cuda/symbols.def")
|
||||
else()
|
||||
message(FATAL_ERROR "onnxruntime_providers_cuda unknown platform, need to specify shared library exports for it")
|
||||
endif()
|
||||
|
||||
install(TARGETS onnxruntime_providers_cuda
|
||||
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||
|
||||
endif()
|
||||
|
||||
if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD
|
||||
|
|
@ -401,6 +431,11 @@ if (onnxruntime_USE_DNNL)
|
|||
set_target_properties(onnxruntime_providers_dnnl PROPERTIES FOLDER "ONNXRuntime")
|
||||
set_target_properties(onnxruntime_providers_dnnl PROPERTIES LINKER_LANGUAGE CXX)
|
||||
|
||||
# Needed for the provider interface, as it includes training headers when training is enabled
|
||||
if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
|
||||
target_include_directories(onnxruntime_providers_dnnl PRIVATE ${ORTTRAINING_ROOT})
|
||||
endif()
|
||||
|
||||
if(APPLE)
|
||||
set_property(TARGET onnxruntime_providers_dnnl APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/core/providers/dnnl/exported_symbols.lst")
|
||||
set_target_properties(onnxruntime_providers_dnnl PROPERTIES
|
||||
|
|
@ -487,6 +522,11 @@ if (onnxruntime_USE_TENSORRT)
|
|||
target_compile_options(onnxruntime_providers_tensorrt INTERFACE /wd4267 /wd4244 /wd4996 /wd4456)
|
||||
endif()
|
||||
|
||||
# Needed for the provider interface, as it includes training headers when training is enabled
|
||||
if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
|
||||
target_include_directories(onnxruntime_providers_tensorrt PRIVATE ${ORTTRAINING_ROOT})
|
||||
endif()
|
||||
|
||||
if(APPLE)
|
||||
set_property(TARGET onnxruntime_providers_tensorrt APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/core/providers/tensorrt/exported_symbols.lst")
|
||||
target_link_libraries(onnxruntime_providers_tensorrt PRIVATE nsync_cpp)
|
||||
|
|
@ -603,6 +643,11 @@ if (onnxruntime_USE_OPENVINO)
|
|||
target_compile_options(onnxruntime_providers_openvino PUBLIC /wd4099 /wd4275 /wd4100 /wd4005 /wd4244 /wd4267)
|
||||
endif()
|
||||
|
||||
# Needed for the provider interface, as it includes training headers when training is enabled
|
||||
if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
|
||||
target_include_directories(onnxruntime_providers_openvino PRIVATE ${ORTTRAINING_ROOT})
|
||||
endif()
|
||||
|
||||
if(APPLE)
|
||||
set_property(TARGET onnxruntime_providers_openvino APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/core/providers/openvino/exported_symbols.lst")
|
||||
elseif(UNIX)
|
||||
|
|
|
|||
|
|
@ -92,7 +92,6 @@ endif()
|
|||
set(onnxruntime_pybind11_state_libs
|
||||
onnxruntime_session
|
||||
${onnxruntime_libs}
|
||||
${PROVIDERS_CUDA}
|
||||
${PROVIDERS_MIGRAPHX}
|
||||
${PROVIDERS_NUPHAR}
|
||||
${PROVIDERS_VITISAI}
|
||||
|
|
@ -370,6 +369,7 @@ add_custom_command(
|
|||
)
|
||||
|
||||
if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD
|
||||
AND NOT onnxruntime_ENABLE_TRAINING
|
||||
AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS"
|
||||
AND NOT (CMAKE_SYSTEM_NAME STREQUAL "Android")
|
||||
AND NOT onnxruntime_BUILD_WEBASSEMBLY)
|
||||
|
|
@ -457,6 +457,16 @@ if (onnxruntime_USE_OPENVINO)
|
|||
)
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_CUDA)
|
||||
add_custom_command(
|
||||
TARGET onnxruntime_pybind11_state POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
$<TARGET_FILE:onnxruntime_providers_cuda>
|
||||
$<TARGET_FILE:onnxruntime_providers_shared>
|
||||
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/capi/
|
||||
)
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_TVM)
|
||||
add_custom_command(
|
||||
TARGET onnxruntime_pybind11_state POST_BUILD
|
||||
|
|
|
|||
|
|
@ -22,6 +22,6 @@ set_target_properties(onnxruntime_session PROPERTIES FOLDER "ONNXRuntime")
|
|||
if (onnxruntime_USE_CUDA)
|
||||
target_include_directories(onnxruntime_session PRIVATE ${onnxruntime_CUDNN_HOME}/include ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
endif()
|
||||
if (onnxruntime_ENABLE_TRAINING)
|
||||
if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS)
|
||||
target_include_directories(onnxruntime_session PRIVATE ${ORTTRAINING_ROOT})
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -36,173 +36,173 @@ if (onnxruntime_USE_NCCL)
|
|||
target_include_directories(onnxruntime_training PRIVATE ${NCCL_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
set_target_properties(onnxruntime_training PROPERTIES FOLDER "ONNXRuntime")
|
||||
source_group(TREE ${ORTTRAINING_ROOT} FILES ${onnxruntime_training_srcs})
|
||||
if (onnxruntime_BUILD_UNIT_TESTS)
|
||||
set_target_properties(onnxruntime_training PROPERTIES FOLDER "ONNXRuntime")
|
||||
source_group(TREE ${ORTTRAINING_ROOT} FILES ${onnxruntime_training_srcs})
|
||||
|
||||
# training runner lib
|
||||
file(GLOB_RECURSE onnxruntime_training_runner_srcs
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/runner/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/runner/*.cc"
|
||||
)
|
||||
# training runner lib
|
||||
file(GLOB_RECURSE onnxruntime_training_runner_srcs
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/runner/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/runner/*.cc"
|
||||
)
|
||||
|
||||
# perf test utils
|
||||
set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest)
|
||||
set(onnxruntime_perf_test_src
|
||||
"${onnxruntime_perf_test_src_dir}/utils.h")
|
||||
# perf test utils
|
||||
set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest)
|
||||
set(onnxruntime_perf_test_src
|
||||
"${onnxruntime_perf_test_src_dir}/utils.h")
|
||||
|
||||
if(WIN32)
|
||||
list(APPEND onnxruntime_perf_test_src
|
||||
"${onnxruntime_perf_test_src_dir}/windows/utils.cc")
|
||||
else ()
|
||||
list(APPEND onnxruntime_perf_test_src
|
||||
"${onnxruntime_perf_test_src_dir}/posix/utils.cc")
|
||||
endif()
|
||||
|
||||
onnxruntime_add_static_library(onnxruntime_training_runner ${onnxruntime_training_runner_srcs} ${onnxruntime_perf_test_src})
|
||||
add_dependencies(onnxruntime_training_runner ${onnxruntime_EXTERNAL_DEPENDENCIES} onnx onnxruntime_providers)
|
||||
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_runner onnxruntime_training onnxruntime_framework onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
|
||||
target_include_directories(onnxruntime_training_runner PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${eigen_INCLUDE_DIRS} PUBLIC ${onnxruntime_graph_header})
|
||||
target_link_libraries(onnxruntime_training_runner PRIVATE nlohmann_json::nlohmann_json)
|
||||
if (onnxruntime_USE_CUDA)
|
||||
target_include_directories(onnxruntime_training_runner PUBLIC ${onnxruntime_CUDNN_HOME}/include ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_NCCL)
|
||||
target_include_directories(onnxruntime_training_runner PRIVATE ${NCCL_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_ROCM)
|
||||
add_definitions(-DUSE_ROCM=1)
|
||||
target_include_directories(onnxruntime_training_runner PUBLIC ${onnxruntime_ROCM_HOME}/include)
|
||||
endif()
|
||||
|
||||
check_cxx_compiler_flag(-Wno-maybe-uninitialized HAS_NO_MAYBE_UNINITIALIZED)
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_runner PUBLIC "-Wno-maybe-uninitialized")
|
||||
if(WIN32)
|
||||
list(APPEND onnxruntime_perf_test_src
|
||||
"${onnxruntime_perf_test_src_dir}/windows/utils.cc")
|
||||
else ()
|
||||
list(APPEND onnxruntime_perf_test_src
|
||||
"${onnxruntime_perf_test_src_dir}/posix/utils.cc")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_ROCM)
|
||||
target_compile_options(onnxruntime_training_runner PUBLIC -D__HIP_PLATFORM_HCC__=1)
|
||||
endif()
|
||||
onnxruntime_add_static_library(onnxruntime_training_runner ${onnxruntime_training_runner_srcs} ${onnxruntime_perf_test_src})
|
||||
add_dependencies(onnxruntime_training_runner ${onnxruntime_EXTERNAL_DEPENDENCIES} onnx onnxruntime_providers)
|
||||
|
||||
set_target_properties(onnxruntime_training_runner PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_training_runner_srcs} ${onnxruntime_perf_test_src})
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_runner onnxruntime_training onnxruntime_framework onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
|
||||
|
||||
# MNIST
|
||||
file(GLOB_RECURSE training_mnist_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/mnist/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/mnist/mnist_data_provider.cc"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/mnist/main.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_mnist ${training_mnist_src})
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_mnist onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_mnist PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
|
||||
set(ONNXRUNTIME_LIBS
|
||||
onnxruntime_session
|
||||
${onnxruntime_libs}
|
||||
${PROVIDERS_CUDA}
|
||||
${PROVIDERS_ROCM}
|
||||
${PROVIDERS_MKLDNN}
|
||||
onnxruntime_optimizer
|
||||
onnxruntime_providers
|
||||
onnxruntime_util
|
||||
onnxruntime_framework
|
||||
onnxruntime_graph
|
||||
onnxruntime_common
|
||||
onnxruntime_mlas
|
||||
onnxruntime_flatbuffers
|
||||
)
|
||||
|
||||
if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
|
||||
list(APPEND ONNXRUNTIME_LIBS onnxruntime_language_interop onnxruntime_pyop)
|
||||
endif()
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_mnist PUBLIC "-Wno-maybe-uninitialized")
|
||||
target_include_directories(onnxruntime_training_runner PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${eigen_INCLUDE_DIRS} PUBLIC ${onnxruntime_graph_header})
|
||||
target_link_libraries(onnxruntime_training_runner PRIVATE nlohmann_json::nlohmann_json)
|
||||
if (onnxruntime_USE_CUDA)
|
||||
target_include_directories(onnxruntime_training_runner PUBLIC ${onnxruntime_CUDNN_HOME}/include ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
endif()
|
||||
endif()
|
||||
target_link_libraries(onnxruntime_training_mnist PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_mnist PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
|
||||
# squeezenet
|
||||
# Disabling build for squeezenet, as no one is using this
|
||||
#[[
|
||||
file(GLOB_RECURSE training_squeezene_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/squeezenet/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/squeezenet/*.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_squeezenet ${training_squeezene_src})
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_squeezenet onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_squeezenet PUBLIC ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${eigen_INCLUDE_DIRS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
if(UNIX AND NOT APPLE)
|
||||
target_compile_options(onnxruntime_training_squeezenet PUBLIC "-Wno-maybe-uninitialized")
|
||||
endif()
|
||||
target_link_libraries(onnxruntime_training_squeezenet PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_squeezenet PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
]]
|
||||
|
||||
# BERT
|
||||
file(GLOB_RECURSE training_bert_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/bert/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/bert/*.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_bert ${training_bert_src})
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_bert PUBLIC "-Wno-maybe-uninitialized")
|
||||
if (onnxruntime_USE_NCCL)
|
||||
target_include_directories(onnxruntime_training_runner PRIVATE ${NCCL_INCLUDE_DIRS})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_bert onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_bert PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${MPI_CXX_INCLUDE_DIRS} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
|
||||
target_link_libraries(onnxruntime_training_bert PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_bert PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
# Pipeline
|
||||
file(GLOB_RECURSE training_pipeline_poc_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/pipeline_poc/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/pipeline_poc/*.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_pipeline_poc ${training_pipeline_poc_src})
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_pipeline_poc PUBLIC "-Wno-maybe-uninitialized")
|
||||
if (onnxruntime_USE_ROCM)
|
||||
add_definitions(-DUSE_ROCM=1)
|
||||
target_include_directories(onnxruntime_training_runner PUBLIC ${onnxruntime_ROCM_HOME}/include)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_pipeline_poc onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_pipeline_poc PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${MPI_CXX_INCLUDE_DIRS} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
if (onnxruntime_USE_NCCL)
|
||||
target_include_directories(onnxruntime_training_pipeline_poc PRIVATE ${NCCL_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
target_link_libraries(onnxruntime_training_pipeline_poc PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_pipeline_poc PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
# GPT-2
|
||||
file(GLOB_RECURSE training_gpt2_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/gpt2/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/gpt2/*.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_gpt2 ${training_gpt2_src})
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_gpt2 PUBLIC "-Wno-maybe-uninitialized")
|
||||
check_cxx_compiler_flag(-Wno-maybe-uninitialized HAS_NO_MAYBE_UNINITIALIZED)
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_runner PUBLIC "-Wno-maybe-uninitialized")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_gpt2 onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_gpt2 PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${MPI_CXX_INCLUDE_DIRS} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
|
||||
target_link_libraries(onnxruntime_training_gpt2 PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_gpt2 PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
if (onnxruntime_USE_ROCM)
|
||||
target_compile_options(onnxruntime_training_runner PUBLIC -D__HIP_PLATFORM_HCC__=1)
|
||||
endif()
|
||||
|
||||
set_target_properties(onnxruntime_training_runner PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_training_runner_srcs} ${onnxruntime_perf_test_src})
|
||||
|
||||
# MNIST
|
||||
file(GLOB_RECURSE training_mnist_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/mnist/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/mnist/mnist_data_provider.cc"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/mnist/main.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_mnist ${training_mnist_src})
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_mnist onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_mnist PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
|
||||
set(ONNXRUNTIME_LIBS
|
||||
onnxruntime_session
|
||||
${onnxruntime_libs}
|
||||
${PROVIDERS_ROCM}
|
||||
${PROVIDERS_MKLDNN}
|
||||
onnxruntime_optimizer
|
||||
onnxruntime_providers
|
||||
onnxruntime_util
|
||||
onnxruntime_framework
|
||||
onnxruntime_graph
|
||||
onnxruntime_common
|
||||
onnxruntime_mlas
|
||||
onnxruntime_flatbuffers
|
||||
)
|
||||
|
||||
if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
|
||||
list(APPEND ONNXRUNTIME_LIBS onnxruntime_language_interop onnxruntime_pyop)
|
||||
endif()
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_mnist PUBLIC "-Wno-maybe-uninitialized")
|
||||
endif()
|
||||
endif()
|
||||
target_link_libraries(onnxruntime_training_mnist PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_mnist PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
# squeezenet
|
||||
# Disabling build for squeezenet, as no one is using this
|
||||
#[[
|
||||
file(GLOB_RECURSE training_squeezene_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/squeezenet/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/squeezenet/*.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_squeezenet ${training_squeezene_src})
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_squeezenet onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_squeezenet PUBLIC ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${eigen_INCLUDE_DIRS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
if(UNIX AND NOT APPLE)
|
||||
target_compile_options(onnxruntime_training_squeezenet PUBLIC "-Wno-maybe-uninitialized")
|
||||
endif()
|
||||
target_link_libraries(onnxruntime_training_squeezenet PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_squeezenet PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
]]
|
||||
|
||||
# BERT
|
||||
file(GLOB_RECURSE training_bert_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/bert/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/bert/*.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_bert ${training_bert_src})
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_bert PUBLIC "-Wno-maybe-uninitialized")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_bert onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_bert PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${MPI_CXX_INCLUDE_DIRS} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
|
||||
target_link_libraries(onnxruntime_training_bert PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_bert PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
# Pipeline
|
||||
file(GLOB_RECURSE training_pipeline_poc_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/pipeline_poc/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/pipeline_poc/*.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_pipeline_poc ${training_pipeline_poc_src})
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_pipeline_poc PUBLIC "-Wno-maybe-uninitialized")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_pipeline_poc onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_pipeline_poc PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${MPI_CXX_INCLUDE_DIRS} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
if (onnxruntime_USE_NCCL)
|
||||
target_include_directories(onnxruntime_training_pipeline_poc PRIVATE ${NCCL_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
target_link_libraries(onnxruntime_training_pipeline_poc PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_pipeline_poc PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
# GPT-2
|
||||
file(GLOB_RECURSE training_gpt2_src
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/gpt2/*.h"
|
||||
"${ORTTRAINING_SOURCE_DIR}/models/gpt2/*.cc"
|
||||
)
|
||||
onnxruntime_add_executable(onnxruntime_training_gpt2 ${training_gpt2_src})
|
||||
if(UNIX AND NOT APPLE)
|
||||
if (HAS_NO_MAYBE_UNINITIALIZED)
|
||||
target_compile_options(onnxruntime_training_gpt2 PUBLIC "-Wno-maybe-uninitialized")
|
||||
endif()
|
||||
endif()
|
||||
onnxruntime_add_include_to_target(onnxruntime_training_gpt2 onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training flatbuffers)
|
||||
target_include_directories(onnxruntime_training_gpt2 PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${MPI_CXX_INCLUDE_DIRS} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner)
|
||||
|
||||
target_link_libraries(onnxruntime_training_gpt2 PRIVATE onnxruntime_training_runner onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES})
|
||||
set_target_properties(onnxruntime_training_gpt2 PROPERTIES FOLDER "ONNXRuntimeTest")
|
||||
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -396,18 +396,6 @@ endif()
|
|||
|
||||
set (onnxruntime_test_providers_dependencies ${onnxruntime_EXTERNAL_DEPENDENCIES})
|
||||
|
||||
if(onnxruntime_USE_CUDA)
|
||||
list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_cuda)
|
||||
endif()
|
||||
|
||||
if(onnxruntime_USE_DNNL)
|
||||
list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_dnnl onnxruntime_providers_shared)
|
||||
endif()
|
||||
|
||||
if(onnxruntime_USE_OPENVINO)
|
||||
list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_openvino onnxruntime_providers_shared)
|
||||
endif()
|
||||
|
||||
if(onnxruntime_USE_NNAPI_BUILTIN)
|
||||
list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_nnapi)
|
||||
endif()
|
||||
|
|
@ -466,8 +454,7 @@ set(ONNXRUNTIME_TEST_LIBS
|
|||
onnxruntime_session
|
||||
${ONNXRUNTIME_INTEROP_TEST_LIBS}
|
||||
${onnxruntime_libs}
|
||||
${PROVIDERS_CUDA}
|
||||
# TENSORRT, DNNL, and OpenVINO are explicitly linked at runtime
|
||||
# CUDA, TENSORRT, DNNL, and OpenVINO are dynamically loaded at runtime
|
||||
${PROVIDERS_MIGRAPHX}
|
||||
${PROVIDERS_NUPHAR}
|
||||
${PROVIDERS_NNAPI}
|
||||
|
|
@ -1139,6 +1126,7 @@ endif()
|
|||
|
||||
# limit to only test on windows first, due to a runtime path issue on linux
|
||||
if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD
|
||||
AND NOT onnxruntime_ENABLE_TRAINING
|
||||
AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS"
|
||||
AND NOT (CMAKE_SYSTEM_NAME STREQUAL "Android")
|
||||
AND NOT onnxruntime_BUILD_WEBASSEMBLY
|
||||
|
|
|
|||
|
|
@ -169,6 +169,7 @@ if (onnxruntime_USE_TELEMETRY)
|
|||
endif()
|
||||
|
||||
# Compiler flags
|
||||
target_compile_definitions(winml_lib_telemetry PRIVATE WINML_ROOT_NS=${winml_root_ns})
|
||||
target_compile_definitions(winml_lib_telemetry PRIVATE PLATFORM_WINDOWS)
|
||||
target_compile_definitions(winml_lib_telemetry PRIVATE _SCL_SECURE_NO_WARNINGS) # remove warnings about unchecked iterators
|
||||
target_compile_definitions(winml_lib_telemetry PRIVATE BINARY_NAME=\"${BINARY_NAME}\")
|
||||
|
|
@ -177,7 +178,10 @@ target_compile_definitions(winml_lib_telemetry PRIVATE BINARY_NAME=\"${BINARY_NA
|
|||
target_precompiled_header(winml_lib_telemetry pch.h)
|
||||
|
||||
# Includes
|
||||
target_include_directories(winml_lib_telemetry PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml/sdk/cppwinrt/include)
|
||||
target_include_directories(winml_lib_telemetry PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) # windows machine learning generated component headers
|
||||
target_include_directories(winml_lib_telemetry PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml_api) # windows machine learning generated component headers
|
||||
target_include_directories(winml_lib_telemetry PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml_api/comp_generated) # windows machine learning generated component headers
|
||||
target_include_directories(winml_lib_telemetry PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/winml/sdk/cppwinrt/include) # sdk cppwinrt headers
|
||||
target_include_directories(winml_lib_telemetry PRIVATE ${CMAKE_SOURCE_DIR}/common/inc)
|
||||
target_include_directories(winml_lib_telemetry PRIVATE ${winml_lib_telemetry_dir})
|
||||
target_include_directories(winml_lib_telemetry PRIVATE ${winml_lib_common_dir}/inc)
|
||||
|
|
@ -189,6 +193,12 @@ set_target_properties(winml_lib_telemetry
|
|||
FOLDER
|
||||
${target_folder})
|
||||
|
||||
# Add deps
|
||||
add_dependencies(winml_lib_telemetry winml_sdk_cppwinrt)
|
||||
add_dependencies(winml_lib_telemetry winml_api)
|
||||
add_dependencies(winml_lib_telemetry winml_api_native)
|
||||
add_dependencies(winml_lib_telemetry winml_api_native_internal)
|
||||
|
||||
# Link libraries
|
||||
target_link_libraries(winml_lib_telemetry PRIVATE wil)
|
||||
|
||||
|
|
|
|||
|
|
@ -180,6 +180,8 @@ class KernelDef {
|
|||
|
||||
class KernelDefBuilder {
|
||||
public:
|
||||
static std::unique_ptr<KernelDefBuilder> Create() { return std::make_unique<KernelDefBuilder>(); }
|
||||
|
||||
explicit KernelDefBuilder()
|
||||
: kernel_def_(new KernelDef()) {}
|
||||
|
||||
|
|
@ -323,6 +325,13 @@ class KernelDefBuilder {
|
|||
return *this;
|
||||
}
|
||||
|
||||
KernelDefBuilder& InputMemoryType(OrtMemType type, const std::vector<int>& input_indexes) {
|
||||
for (auto input_index : input_indexes) {
|
||||
kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, type));
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
/**
|
||||
Specify that this kernel provides an output arg
|
||||
in certain memory type (instead of the default, device memory).
|
||||
|
|
|
|||
|
|
@ -4,11 +4,13 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/common/exceptions.h"
|
||||
#include "core/framework/allocator.h"
|
||||
#include "core/framework/data_types.h"
|
||||
#include "core/framework/tensor.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
class SparseTensor;
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/status.h"
|
||||
#include "core/graph/graph_viewer.h"
|
||||
#include "gsl/gsl"
|
||||
#endif
|
||||
|
||||
#ifdef __has_attribute
|
||||
#define ORT_HAVE_ATTRIBUTE(x) __has_attribute(x)
|
||||
|
|
|
|||
|
|
@ -34,6 +34,13 @@ namespace onnxruntime {
|
|||
*/
|
||||
class Tensor final {
|
||||
public:
|
||||
static std::unique_ptr<Tensor> Create(MLDataType p_type, const TensorShape& shape, std::shared_ptr<IAllocator> allocator) {
|
||||
return std::make_unique<Tensor>(p_type, shape, allocator);
|
||||
}
|
||||
static std::unique_ptr<Tensor> Create(MLDataType p_type, const TensorShape& shape, void* p_data, const OrtMemoryInfo& alloc, ptrdiff_t offset = 0) {
|
||||
return std::make_unique<Tensor>(p_type, shape, p_data, alloc, offset);
|
||||
}
|
||||
|
||||
Tensor() = default; // to allow creating vector<Tensor> to support seq(tensor)
|
||||
|
||||
/**
|
||||
|
|
@ -235,4 +242,4 @@ class Tensor final {
|
|||
#ifdef __GNUC__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
} // namespace onnxruntime
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -4,6 +4,50 @@
|
|||
#include "onnxruntime_c_api.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
#include "core/framework/provider_options.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
class IAllocator;
|
||||
class IDataTransfer;
|
||||
struct IExecutionProviderFactory;
|
||||
struct CUDAExecutionProviderInfo;
|
||||
enum class ArenaExtendStrategy : int32_t;
|
||||
struct CUDAExecutionProviderExternalAllocatorInfo;
|
||||
|
||||
namespace cuda {
|
||||
class INcclService;
|
||||
}
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
||||
struct ProviderInfo_CUDA {
|
||||
virtual OrtStatus* SetCurrentGpuDeviceId(_In_ int device_id) = 0;
|
||||
virtual OrtStatus* GetCurrentGpuDeviceId(_In_ int* device_id) = 0;
|
||||
|
||||
virtual std::unique_ptr<onnxruntime::IAllocator> CreateCUDAAllocator(int16_t device_id, const char* name) = 0;
|
||||
virtual std::unique_ptr<onnxruntime::IAllocator> CreateCUDAPinnedAllocator(int16_t device_id, const char* name) = 0;
|
||||
virtual std::unique_ptr<onnxruntime::IDataTransfer> CreateGPUDataTransfer(void* stream) = 0;
|
||||
|
||||
virtual void cuda__Impl_Cast(void* stream, const int64_t* input_data, int32_t* output_data, size_t count) = 0;
|
||||
virtual void cuda__Impl_Cast(void* stream, const int32_t* input_data, int64_t* output_data, size_t count) = 0;
|
||||
|
||||
virtual bool CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0;
|
||||
virtual bool CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0;
|
||||
|
||||
virtual void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location) = 0;
|
||||
virtual void cudaMemcpy_HostToDevice(void* dst, const void* src, size_t count) = 0;
|
||||
virtual void cudaMemcpy_DeviceToHost(void* dst, const void* src, size_t count) = 0;
|
||||
virtual int cudaGetDeviceCount() = 0;
|
||||
virtual void CUDAExecutionProviderInfo__FromProviderOptions(const onnxruntime::ProviderOptions& options, onnxruntime::CUDAExecutionProviderInfo& info) = 0;
|
||||
|
||||
#if defined(USE_CUDA) && defined(ORT_USE_NCCL) && defined(USE_NCCL_P2P)
|
||||
virtual onnxruntime::cuda::INcclService& GetINcclService() = 0;
|
||||
#endif
|
||||
|
||||
virtual std::shared_ptr<onnxruntime::IExecutionProviderFactory> CreateExecutionProviderFactory(const onnxruntime::CUDAExecutionProviderInfo& info) = 0;
|
||||
virtual std::shared_ptr<onnxruntime::IAllocator> CreateCudaAllocator(int16_t device_id, size_t gpu_mem_limit, onnxruntime::ArenaExtendStrategy arena_extend_strategy, onnxruntime::CUDAExecutionProviderExternalAllocatorInfo& external_allocator_info, OrtArenaCfg* default_memory_arena_cfg) = 0;
|
||||
};
|
||||
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -338,7 +338,7 @@ struct OrtApiBase {
|
|||
const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; // Pass in ORT_API_VERSION
|
||||
// nullptr will be returned if the version is unsupported, for example when using a runtime older than this header file
|
||||
|
||||
const char*(ORT_API_CALL* GetVersionString)() NO_EXCEPTION;
|
||||
const char*(ORT_API_CALL* GetVersionString)(void)NO_EXCEPTION;
|
||||
};
|
||||
typedef struct OrtApiBase OrtApiBase;
|
||||
|
||||
|
|
@ -1301,16 +1301,16 @@ struct OrtApi {
|
|||
* \param arena_config_values - values to configure the arena
|
||||
* \param num_keys - number of keys passed in
|
||||
* Supported keys are (See docs/C_API.md for details on what the following parameters mean and how to choose these values.):
|
||||
* "max_mem": Maximum memory that can be allocated by the arena based allocator.
|
||||
* "max_mem": Maximum memory that can be allocated by the arena based allocator.
|
||||
Use 0 for ORT to pick the best value. Default is 0.
|
||||
* "arena_extend_strategy": 0 = kNextPowerOfTwo, 1 = kSameAsRequested.
|
||||
* "arena_extend_strategy": 0 = kNextPowerOfTwo, 1 = kSameAsRequested.
|
||||
Use -1 to allow ORT to choose the default.
|
||||
* "initial_chunk_size_bytes": (Possible) Size of the first allocation in the arena.
|
||||
* "initial_chunk_size_bytes": (Possible) Size of the first allocation in the arena.
|
||||
Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default.
|
||||
Ultimately, the first allocation size is determined by the allocation memory request.
|
||||
* "max_dead_bytes_per_chunk": Threshold of unused memory in an allocated chunk of arena memory after
|
||||
Ultimately, the first allocation size is determined by the allocation memory request.
|
||||
* "max_dead_bytes_per_chunk": Threshold of unused memory in an allocated chunk of arena memory after
|
||||
crossing which the current chunk is chunked into 2.
|
||||
* "initial_growth_chunk_size_bytes": (Possible) Size of the second allocation in the arena.
|
||||
* "initial_growth_chunk_size_bytes": (Possible) Size of the second allocation in the arena.
|
||||
Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default.
|
||||
Ultimately, the allocation size is determined by the allocation memory request.
|
||||
Further allocation sizes are governed by the arena extend strategy.
|
||||
|
|
@ -1332,10 +1332,10 @@ struct OrtApi {
|
|||
/*
|
||||
* Creates an OrtPrepackedWeightsContainer instance.
|
||||
* This container will hold pre-packed buffers of shared initializers for sharing between sessions
|
||||
* (i.e.) if there are shared initializers that can be shared between sessions, the pre-packed buffers
|
||||
* (i.e.) if there are shared initializers that can be shared between sessions, the pre-packed buffers
|
||||
* of these (if any) may possibly be shared to provide memory footprint savings. Pass this container
|
||||
* to sessions that you would like to share pre-packed buffers of shared initializers at session
|
||||
* creation time.
|
||||
* to sessions that you would like to share pre-packed buffers of shared initializers at session
|
||||
* creation time.
|
||||
* \out - created OrtPrepackedWeightsContainer instance
|
||||
*/
|
||||
ORT_API2_STATUS(CreatePrepackedWeightsContainer, _Outptr_ OrtPrepackedWeightsContainer** out);
|
||||
|
|
@ -1350,7 +1350,7 @@ struct OrtApi {
|
|||
* Same functionality offered by CreateSession() API except that a container that contains
|
||||
pre-packed weights' buffers is written into/read from by the created session.
|
||||
This is useful when used in conjunction with the AddInitializer() API which injects
|
||||
shared initializer info into sessions. Wherever possible, the pre-packed versions of these
|
||||
shared initializer info into sessions. Wherever possible, the pre-packed versions of these
|
||||
shared initializers are cached in this container so that multiple sessions can just re-use
|
||||
these instead of duplicating these in memory.
|
||||
* \env - OrtEnv instance instance
|
||||
|
|
@ -1367,7 +1367,7 @@ struct OrtApi {
|
|||
* Same functionality offered by CreateSessionFromArray() API except that a container that contains
|
||||
pre-packed weights' buffers is written into/read from by the created session.
|
||||
This is useful when used in conjunction with the AddInitializer() API which injects
|
||||
shared initializer info into sessions. Wherever possible, the pre-packed versions of these
|
||||
shared initializer info into sessions. Wherever possible, the pre-packed versions of these
|
||||
shared initializers are cached in this container so that multiple sessions can just re-use
|
||||
these instead of duplicating these in memory.
|
||||
* \env - OrtEnv instance instance
|
||||
|
|
|
|||
|
|
@ -569,7 +569,6 @@ struct ArenaCfg : Base<OrtArenaCfg> {
|
|||
* \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested
|
||||
* \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default
|
||||
* \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default
|
||||
* \return an instance of ArenaCfg
|
||||
* See docs/C_API.md for details on what the following parameters mean and how to choose these values
|
||||
*/
|
||||
ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk);
|
||||
|
|
|
|||
|
|
@ -36,6 +36,11 @@ final class OnnxRuntime {
|
|||
/** The short name of the ONNX runtime JNI shared library */
|
||||
static final String ONNXRUNTIME_JNI_LIBRARY_NAME = "onnxruntime4j_jni";
|
||||
|
||||
/** The short name of the ONNX runtime shared provider library */
|
||||
static final String ONNXRUNTIME_LIBRARY_SHARED_NAME = "onnxruntime_providers_shared";
|
||||
/** The short name of the ONNX runtime cuda provider library */
|
||||
static final String ONNXRUNTIME_LIBRARY_CUDA_NAME = "onnxruntime_providers_cuda";
|
||||
|
||||
private static final String OS_ARCH_STR = initOsArch();
|
||||
|
||||
private static boolean loaded = false;
|
||||
|
|
@ -93,14 +98,19 @@ final class OnnxRuntime {
|
|||
}
|
||||
Path tempDirectory = isAndroid() ? null : Files.createTempDirectory("onnxruntime-java");
|
||||
try {
|
||||
load(tempDirectory, ONNXRUNTIME_LIBRARY_NAME);
|
||||
load(tempDirectory, ONNXRUNTIME_JNI_LIBRARY_NAME);
|
||||
// Extract and prepare the shared provider libraries but don't try to load them, Onnxruntime
|
||||
// itself will load them
|
||||
load(tempDirectory, ONNXRUNTIME_LIBRARY_SHARED_NAME, false);
|
||||
load(tempDirectory, ONNXRUNTIME_LIBRARY_CUDA_NAME, false);
|
||||
|
||||
load(tempDirectory, ONNXRUNTIME_LIBRARY_NAME, true);
|
||||
load(tempDirectory, ONNXRUNTIME_JNI_LIBRARY_NAME, true);
|
||||
ortApiHandle = initialiseAPIBase(ORT_API_VERSION_7);
|
||||
providers = initialiseProviders(ortApiHandle);
|
||||
loaded = true;
|
||||
} finally {
|
||||
if (tempDirectory != null) {
|
||||
cleanUp(tempDirectory.toFile());
|
||||
cleanUp(tempDirectory.toFile(), false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -110,13 +120,14 @@ final class OnnxRuntime {
|
|||
* in time.
|
||||
*
|
||||
* @param file The file to remove.
|
||||
* @param onExitOnly Delete the file on exit only, vs trying to do it immediately
|
||||
*/
|
||||
private static void cleanUp(File file) {
|
||||
private static void cleanUp(File file, boolean onExitOnly) {
|
||||
if (!file.exists()) {
|
||||
return;
|
||||
}
|
||||
logger.log(Level.FINE, "Deleting " + file);
|
||||
if (!file.delete()) {
|
||||
if (onExitOnly || !file.delete()) {
|
||||
logger.log(Level.FINE, "Deleting " + file + " on exit");
|
||||
file.deleteOnExit();
|
||||
}
|
||||
|
|
@ -136,9 +147,11 @@ final class OnnxRuntime {
|
|||
*
|
||||
* @param tempDirectory The temp directory to write the library resource to.
|
||||
* @param library The bare name of the library.
|
||||
* @param systemLoad If system.Load(..) should be called on the library vs just preparing it
|
||||
* @throws IOException If the file failed to read or write.
|
||||
*/
|
||||
private static void load(Path tempDirectory, String library) throws IOException {
|
||||
private static void load(Path tempDirectory, String library, boolean systemLoad)
|
||||
throws IOException {
|
||||
// On Android, we simply use System.loadLibrary
|
||||
if (isAndroid()) {
|
||||
System.loadLibrary("onnxruntime4j_jni");
|
||||
|
|
@ -180,6 +193,9 @@ final class OnnxRuntime {
|
|||
try (InputStream is = OnnxRuntime.class.getResourceAsStream(resourcePath)) {
|
||||
if (is == null) {
|
||||
// 3a) Not found in resources, load from library path
|
||||
if (!systemLoad) {
|
||||
return; // Failure is expected for optional components we don't need to load
|
||||
}
|
||||
logger.log(
|
||||
Level.FINE, "Attempting to load native library '" + library + "' from library path");
|
||||
System.loadLibrary(library);
|
||||
|
|
@ -201,11 +217,15 @@ final class OnnxRuntime {
|
|||
os.write(buffer, 0, readBytes);
|
||||
}
|
||||
}
|
||||
System.load(tempFile.getAbsolutePath());
|
||||
logger.log(Level.FINE, "Loaded native library '" + library + "' from resource path");
|
||||
if (systemLoad) {
|
||||
System.load(tempFile.getAbsolutePath());
|
||||
logger.log(Level.FINE, "Loaded native library '" + library + "' from resource path");
|
||||
} else {
|
||||
logger.log(Level.FINE, "Extracted native library '" + library + "' from resource path");
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
cleanUp(tempFile);
|
||||
cleanUp(tempFile, !systemLoad);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ Node.js v12+ (recommended v14+)
|
|||
./build.sh --config Release --build_wasm --skip_tests --disable_wasm_exception_catching --disable_rtti
|
||||
```
|
||||
|
||||
To build with multi-thread support, append flag ` --enable_wasm_threads` to the command.
|
||||
To build with multi-thread support, append flag ` --enable_wasm_threads` to the command. Make sure to build both single-thread and multi-thread before next step.
|
||||
|
||||
3. Copy following files from build output folder to `<ORT_ROOT>/js/web/dist/`:
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ Install the latest stable version:
|
|||
npm install onnxruntime-node
|
||||
```
|
||||
|
||||
Refer to [Node.js samples](../../samples/nodejs/README.md) for samples and tutorials.
|
||||
Refer to [ONNX Runtime JavaScript examples](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/js) for samples and tutorials.
|
||||
|
||||
## Requirements
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "onnxruntime-react-native",
|
||||
"version": "1.7.0",
|
||||
"version": "1.8.0",
|
||||
"description": "Onnxruntime bridge for react native",
|
||||
"main": "dist/commonjs/index",
|
||||
"module": "dist/module/index",
|
||||
|
|
|
|||
|
|
@ -16,105 +16,17 @@ ONNX Runtime Web can run on both CPU and GPU. For running on CPU, [WebAssembly](
|
|||
|
||||
See [Compatibility](#Compatibility) and [Operators Supported](#Operators) for a list of platforms and operators ONNX Runtime Web currently supports.
|
||||
|
||||
## Getting Started
|
||||
## Usage
|
||||
|
||||
There are multiple ways to use ONNX Runtime Web in a project:
|
||||
|
||||
### Using `<script>` tag
|
||||
|
||||
This is the most straightforward way to use ONNX Runtime Web. The following HTML example shows how to use it:
|
||||
|
||||
```html
|
||||
<html>
|
||||
<head> </head>
|
||||
|
||||
<body>
|
||||
<!-- Load ONNX Runtime Web -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.min.js"></script>
|
||||
<!-- Code that consume ONNX Runtime Web -->
|
||||
<script>
|
||||
async function runMyModel() {
|
||||
// create a session
|
||||
const myOrtSession = await ort.InferenceSession.create(
|
||||
"./my-model.onnx"
|
||||
);
|
||||
// generate model input
|
||||
const input0 = new ort.Tensor(
|
||||
new Float32Array([1.0, 2.0, 3.0, 4.0]) /* data */,
|
||||
[2, 2] /* dims */
|
||||
);
|
||||
// execute the model
|
||||
const outputs = await myOrtSession.run({ input_0: input0 });
|
||||
// consume the output
|
||||
const outputTensor = outputs["output_0"];
|
||||
console.log(`model output tensor: ${outputTensor.data}.`);
|
||||
}
|
||||
runMyModel();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
```
|
||||
|
||||
<!-- TODO: Refer to [browser/Add](./examples/browser/add) for an example. -->
|
||||
|
||||
### Using NPM and bundling tools
|
||||
|
||||
Modern browser based applications are usually built by frameworks like [Angular](https://angular.io/), [React](https://reactjs.org/), [Vue.js](https://vuejs.org/) and so on. This solution usually builds the source code into one or more bundle file(s). The following TypeScript example shows how to use ONNX Runtime Web in an async context:
|
||||
|
||||
1. Import `Tensor` and `InferenceSession`.
|
||||
|
||||
```ts
|
||||
import { Tensor, InferenceSession } from "onnxruntime-web";
|
||||
```
|
||||
|
||||
2. Create an instance of `InferenceSession` and load ONNX model.
|
||||
|
||||
```ts
|
||||
// use the following in an async method
|
||||
const url = "./data/models/resnet/model.onnx";
|
||||
const session = await InferenceSession.create(url);
|
||||
```
|
||||
|
||||
3. Create your input Tensor(s) similar to the example below. You need to do any pre-processing required by
|
||||
your model at this stage. For that refer to the documentation of the model you have:
|
||||
|
||||
```javascript
|
||||
// creating an array of input Tensors is the easiest way. For other options see the API documentation
|
||||
const input0 = new Tensor(new Float32Array([1.0, 2.0, 3.0, 4.0]), [2, 2]);
|
||||
```
|
||||
|
||||
4. Run the model with the input Tensors. The output Tensor(s) are available once the run operation is complete:
|
||||
|
||||
```javascript
|
||||
// run this in an async method:
|
||||
// assume model's input name is 'input_0' and output name is 'output_0'
|
||||
const outputs = await session.run({ input_0: input0 });
|
||||
const outputTensor = outputs.output_0;
|
||||
```
|
||||
|
||||
5. Bundle your code. All web application frameworks offer bundling tools and instructions. Specifically, you can specify onnxruntime-web as an external dependency:
|
||||
|
||||
```js
|
||||
// a webpack example
|
||||
externals: {
|
||||
'onnxruntime-web': 'ort', // add this line in your webpack.config.js
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
so that you can consume the file `ort.min.js` from a CDN provider demonstrated as above.
|
||||
|
||||
<!-- TODO More verbose examples on how to use ONNX Runtime Web are located under the `examples` folder. For further info see [Examples](./examples/README.md) -->
|
||||
Refer to [ONNX Runtime JavaScript examples](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/js) for samples and tutorials.
|
||||
|
||||
## Documents
|
||||
|
||||
### Developers
|
||||
|
||||
<!-- TODO development documents and API -->
|
||||
Refer to [Using VSCode](../README.md#Using-VSCode) for setting up development environment.
|
||||
|
||||
For information on ONNX.js development, please check [Development](./docs/development.md)
|
||||
|
||||
For API reference, please check [API](./docs/api.md).
|
||||
For information about building ONNX Runtime Web development, please check [Build](../README.md#build-2).
|
||||
|
||||
### Getting ONNX models
|
||||
|
||||
|
|
|
|||
2
js/web/package-lock.json
generated
2
js/web/package-lock.json
generated
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "onnxruntime-web",
|
||||
"version": "1.7.0",
|
||||
"version": "1.8.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
"type": "git"
|
||||
},
|
||||
"author": "fs-eire",
|
||||
"version": "1.7.0",
|
||||
"version": "1.8.0",
|
||||
"keywords": [
|
||||
"ONNX",
|
||||
"ONNXRuntime",
|
||||
|
|
@ -73,12 +73,5 @@
|
|||
"jsdelivr": "dist/ort.min.js",
|
||||
"unpkg": "dist/ort.min.js",
|
||||
"module": "./lib/index.js",
|
||||
"browser": {
|
||||
"fs": false,
|
||||
"path": false,
|
||||
"util": false,
|
||||
"worker_threads": false,
|
||||
"perf_hooks": false,
|
||||
"os": false
|
||||
}
|
||||
"browser": "dist/ort-web.min.js"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,20 @@ function buildConfig({ filename, format, target, mode, devtool }) {
|
|||
type: format
|
||||
}
|
||||
},
|
||||
resolve: { extensions: ['.ts', '.js'] },
|
||||
resolve: {
|
||||
extensions: ['.ts', '.js'],
|
||||
alias: {
|
||||
"util": false,
|
||||
},
|
||||
fallback: {
|
||||
"fs": false,
|
||||
"path": false,
|
||||
"util": false,
|
||||
"os": false,
|
||||
"worker_threads": false,
|
||||
"perf_hooks": false,
|
||||
}
|
||||
},
|
||||
plugins: [new webpack.WatchIgnorePlugin({ paths: [/\.js$/, /\.d\.ts$/] })],
|
||||
module: {
|
||||
rules: [{
|
||||
|
|
@ -57,7 +70,7 @@ function buildConfig({ filename, format, target, mode, devtool }) {
|
|||
}
|
||||
]
|
||||
}, {
|
||||
test: /\.worker.js$/,
|
||||
test: /ort-wasm.*\.worker\.js$/,
|
||||
type: 'asset/source'
|
||||
}]
|
||||
},
|
||||
|
|
@ -66,10 +79,9 @@ function buildConfig({ filename, format, target, mode, devtool }) {
|
|||
};
|
||||
|
||||
if (mode === 'production') {
|
||||
config.resolve.alias = {
|
||||
'./binding/ort-wasm-threaded.js': './binding/ort-wasm-threaded.min.js',
|
||||
'./binding/ort-wasm-threaded.worker.js': './binding/ort-wasm-threaded.min.worker.js'
|
||||
};
|
||||
config.resolve.alias['./binding/ort-wasm-threaded.js'] = './binding/ort-wasm-threaded.min.js';
|
||||
config.resolve.alias['./binding/ort-wasm-threaded.worker.js'] = './binding/ort-wasm-threaded.min.worker.js';
|
||||
|
||||
const options = defaultTerserPluginOptions();
|
||||
options.terserOptions.format.preamble = COPYRIGHT_BANNER;
|
||||
config.plugins.push(new TerserPlugin(options));
|
||||
|
|
@ -90,8 +102,6 @@ function buildOrtConfig({
|
|||
const config = buildConfig({ filename: `ort${suffix}.js`, format: 'umd', target, mode, devtool });
|
||||
// set global name 'ort'
|
||||
config.output.library.name = 'ort';
|
||||
// do not use those node builtin modules in browser
|
||||
config.resolve.fallback = { path: false, fs: false, util: false };
|
||||
return config;
|
||||
}
|
||||
|
||||
|
|
@ -150,8 +160,11 @@ function buildTestRunnerConfig({
|
|||
'../../node': '../../node'
|
||||
},
|
||||
resolve: {
|
||||
alias: {
|
||||
// make sure to refer to original source files instead of generated bundle in test-main.
|
||||
'..$': '../lib/index'
|
||||
},
|
||||
extensions: ['.ts', '.js'],
|
||||
aliasFields: [],
|
||||
fallback: {
|
||||
'./binding/ort-wasm.js': false,
|
||||
'./binding/ort-wasm-threaded.js': false,
|
||||
|
|
@ -174,7 +187,7 @@ function buildTestRunnerConfig({
|
|||
}
|
||||
]
|
||||
}, {
|
||||
test: /\.worker\.js$/,
|
||||
test: /ort-wasm.*\.worker\.js$/,
|
||||
type: 'asset/source'
|
||||
}]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -47,14 +47,6 @@ ONNX_OPERATOR_TYPED_KERNEL_EX(
|
|||
.TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
|
||||
Attention<float>);
|
||||
|
||||
AttentionBase::AttentionBase(const OpKernelInfo& info) {
|
||||
int64_t num_heads = 0;
|
||||
ORT_ENFORCE(info.GetAttr("num_heads", &num_heads).IsOK() && num_heads > 0);
|
||||
num_heads_ = static_cast<int>(num_heads);
|
||||
|
||||
is_unidirectional_ = info.GetAttrOrDefault<int64_t>("unidirectional", 0) == 1;
|
||||
}
|
||||
|
||||
Status AttentionBase::CheckInputs(const TensorShape& input_shape,
|
||||
const TensorShape& weights_shape,
|
||||
const TensorShape& bias_shape,
|
||||
|
|
|
|||
|
|
@ -10,15 +10,7 @@ namespace onnxruntime {
|
|||
namespace contrib {
|
||||
|
||||
class AttentionBase {
|
||||
protected:
|
||||
AttentionBase(const OpKernelInfo& info);
|
||||
|
||||
Status CheckInputs(const TensorShape& input_shape,
|
||||
const TensorShape& weights_shape,
|
||||
const TensorShape& bias_shape,
|
||||
const Tensor*& mask_index, // For dummy mask with shape (1, 1) or (batch_size, 1), it will be updated to nullptr.
|
||||
const Tensor* past) const;
|
||||
|
||||
public:
|
||||
// This check function is specifically used in cuda
|
||||
Status CheckInputs(const TensorShape& input_shape,
|
||||
const TensorShape& weights_shape,
|
||||
|
|
@ -34,6 +26,21 @@ class AttentionBase {
|
|||
int sequence_length,
|
||||
int& past_sequence_length) const;
|
||||
|
||||
protected:
|
||||
AttentionBase(const OpKernelInfo& info) {
|
||||
int64_t num_heads = 0;
|
||||
ORT_ENFORCE(info.GetAttr("num_heads", &num_heads).IsOK() && num_heads > 0);
|
||||
num_heads_ = static_cast<int>(num_heads);
|
||||
|
||||
is_unidirectional_ = info.GetAttrOrDefault<int64_t>("unidirectional", 0) == 1;
|
||||
}
|
||||
|
||||
Status CheckInputs(const TensorShape& input_shape,
|
||||
const TensorShape& weights_shape,
|
||||
const TensorShape& bias_shape,
|
||||
const Tensor*& mask_index, // For dummy mask with shape (1, 1) or (batch_size, 1), it will be updated to nullptr.
|
||||
const Tensor* past) const;
|
||||
|
||||
int num_heads_; // number of attention heads
|
||||
bool is_unidirectional_; // whether every token can only attend to previous tokens.
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,19 +5,33 @@
|
|||
#include "core/framework/tensorprotoutils.h"
|
||||
#include "onnx/defs/tensor_proto_util.h"
|
||||
|
||||
#include "longformer_attention_base.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace contrib {
|
||||
|
||||
Status LongformerAttentionBase__CheckInputs(const LongformerAttentionBase* p,
|
||||
const TensorShape& input_shape,
|
||||
const TensorShape& weights_shape,
|
||||
const TensorShape& bias_shape,
|
||||
const TensorShape& mask_shape,
|
||||
const TensorShape& global_weights_shape,
|
||||
const TensorShape& global_bias_shape,
|
||||
const TensorShape& global_shape) {
|
||||
return p->CheckInputs(input_shape, weights_shape, bias_shape, mask_shape, global_weights_shape, global_bias_shape, global_shape);
|
||||
}
|
||||
|
||||
namespace embed_layer_norm {
|
||||
|
||||
Status CheckInputs(const OpKernelContext* context) {
|
||||
const Tensor* input_ids = context->Input<Tensor>(0);
|
||||
const Tensor* segment_ids = context->Input<Tensor>(1); // optional. nullptr if it's distill-bert
|
||||
const Tensor* segment_ids = context->Input<Tensor>(1); // optional. nullptr if it's distill-bert
|
||||
const Tensor* word_embedding = context->Input<Tensor>(2);
|
||||
const Tensor* position_embedding = context->Input<Tensor>(3);
|
||||
const Tensor* segment_embedding = context->Input<Tensor>(4); // optional. nullptr if it's distill-bert
|
||||
const Tensor* segment_embedding = context->Input<Tensor>(4); // optional. nullptr if it's distill-bert
|
||||
const Tensor* gamma = context->Input<Tensor>(5);
|
||||
const Tensor* beta = context->Input<Tensor>(6);
|
||||
const Tensor* mask = context->Input<Tensor>(7); // optional. nullptr if not provided
|
||||
const Tensor* mask = context->Input<Tensor>(7); // optional. nullptr if not provided
|
||||
|
||||
if (nullptr != segment_ids && input_ids->Shape() != segment_ids->Shape()) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
|
||||
|
|
@ -29,7 +43,6 @@ Status CheckInputs(const OpKernelContext* context) {
|
|||
"Input 0 and 7 (mask) shall have same shape");
|
||||
}
|
||||
|
||||
|
||||
const auto& input_dims = input_ids->Shape().GetDims();
|
||||
if (input_dims.size() != 2) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace contrib {
|
||||
|
|
|
|||
|
|
@ -6,16 +6,6 @@
|
|||
namespace onnxruntime {
|
||||
namespace contrib {
|
||||
|
||||
LongformerAttentionBase::LongformerAttentionBase(const OpKernelInfo& info) {
|
||||
int64_t num_heads = 0;
|
||||
ORT_ENFORCE(info.GetAttr("num_heads", &num_heads).IsOK() && num_heads > 0);
|
||||
num_heads_ = static_cast<int>(num_heads);
|
||||
|
||||
int64_t window = 0;
|
||||
ORT_ENFORCE(info.GetAttr("window", &window).IsOK() && window > 0);
|
||||
window_ = static_cast<int>(window);
|
||||
}
|
||||
|
||||
Status LongformerAttentionBase::CheckInputs(const TensorShape& input_shape,
|
||||
const TensorShape& weights_shape,
|
||||
const TensorShape& bias_shape,
|
||||
|
|
|
|||
|
|
@ -10,9 +10,7 @@ namespace onnxruntime {
|
|||
namespace contrib {
|
||||
|
||||
class LongformerAttentionBase {
|
||||
protected:
|
||||
LongformerAttentionBase(const OpKernelInfo& info);
|
||||
|
||||
public:
|
||||
Status CheckInputs(const TensorShape& input_shape,
|
||||
const TensorShape& weights_shape,
|
||||
const TensorShape& bias_shape,
|
||||
|
|
@ -21,6 +19,17 @@ class LongformerAttentionBase {
|
|||
const TensorShape& global_bias_shape,
|
||||
const TensorShape& global_shape) const;
|
||||
|
||||
protected:
|
||||
LongformerAttentionBase(const OpKernelInfo& info) {
|
||||
int64_t num_heads = 0;
|
||||
ORT_ENFORCE(info.GetAttr("num_heads", &num_heads).IsOK() && num_heads > 0);
|
||||
num_heads_ = static_cast<int>(num_heads);
|
||||
|
||||
int64_t window = 0;
|
||||
ORT_ENFORCE(info.GetAttr("window", &window).IsOK() && window > 0);
|
||||
window_ = static_cast<int>(window);
|
||||
}
|
||||
|
||||
int num_heads_; // Number of attention heads
|
||||
int window_; // Attention windows length (W). It is half (one-sided) of total window size.
|
||||
};
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ namespace cuda {
|
|||
ver, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()) \
|
||||
.MayInplace(0, 0), \
|
||||
x<T>);
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#include "attention.h"
|
||||
#include "core/framework/tensorprotoutils.h"
|
||||
#include "attention_impl.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "core/providers/cuda/shared_inc/fpgeneric.h"
|
||||
#include "attention_impl.h"
|
||||
|
||||
using namespace onnxruntime::cuda;
|
||||
using namespace ::onnxruntime::common;
|
||||
|
|
@ -22,7 +21,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
Attention<T>);
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/providers/cuda/cuda_kernel.h"
|
||||
#include "contrib_ops/cpu/bert/attention_base.h"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/providers/common.h"
|
||||
#include "core/framework/tensorprotoutils.h"
|
||||
#include "onnx/defs/tensor_proto_util.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "contrib_ops/cpu/bert/embed_layer_norm_helper.h"
|
||||
#include "embed_layer_norm.h"
|
||||
#include "embed_layer_norm_impl.h"
|
||||
|
|
@ -19,7 +17,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
EmbedLayerNorm<T>);
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
#include "core/common/common.h"
|
||||
#include "core/providers/cuda/cuda_kernel.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
@ -16,6 +15,7 @@ class EmbedLayerNorm final : public CudaKernel {
|
|||
public:
|
||||
EmbedLayerNorm(const OpKernelInfo& op_kernel_info);
|
||||
Status ComputeInternal(OpKernelContext* ctx) const override;
|
||||
|
||||
private:
|
||||
float epsilon_;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/providers/common.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "core/providers/cuda/cudnn_common.h"
|
||||
#include "core/framework/tensorprotoutils.h"
|
||||
#include "fast_gelu.h"
|
||||
#include "fast_gelu_impl.h"
|
||||
#include "contrib_ops/cpu/bert/bias_gelu_helper.h"
|
||||
|
|
@ -19,7 +18,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
FastGelu<T>);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "longformer_attention.h"
|
||||
#include "core/framework/tensorprotoutils.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "core/providers/cuda/shared_inc/fpgeneric.h"
|
||||
#include "core/platform/env_var_utils.h"
|
||||
#include "longformer_attention.h"
|
||||
#include "longformer_global_impl.h"
|
||||
#include "longformer_attention_impl.h"
|
||||
|
||||
|
|
@ -24,7 +23,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
LongformerAttention<T>);
|
||||
|
||||
|
|
@ -45,6 +44,7 @@ class AutoDestoryCudaEvent {
|
|||
cudaEvent_t& Get() {
|
||||
return cuda_event_;
|
||||
}
|
||||
|
||||
private:
|
||||
cudaEvent_t cuda_event_;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/providers/common.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "core/framework/tensorprotoutils.h"
|
||||
#include "onnx/defs/tensor_proto_util.h"
|
||||
#include "contrib_ops/cuda/bert/skip_layer_norm.h"
|
||||
#include "contrib_ops/cuda/bert/skip_layer_norm_impl.h"
|
||||
#include "skip_layer_norm.h"
|
||||
#include "skip_layer_norm_impl.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace contrib {
|
||||
|
|
@ -19,7 +16,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
SkipLayerNorm<T>);
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@ ONNX_OPERATOR_TYPED_KERNEL_EX(
|
|||
1,
|
||||
float,
|
||||
kCudaExecutionProvider,
|
||||
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()).InputMemoryType<OrtMemTypeCPUInput>(2),
|
||||
(*KernelDefBuilder::Create())
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<float>())
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 2),
|
||||
ConvTransposeWithDynamicPads<float>);
|
||||
} // namespace cuda
|
||||
} // namespace contrib
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/providers/shared_library/provider_api.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "core/framework/kernel_registry.h"
|
||||
|
||||
using namespace onnxruntime::common;
|
||||
|
||||
|
|
@ -17,9 +17,9 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1
|
|||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, BiasGelu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, BiasGelu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, BiasGelu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, TransposeMatMul); // backward compatibility
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, TransposeMatMul); // backward compatibility
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, TransposeMatMul); // backward compatibility
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, TransposeMatMul); // backward compatibility
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, TransposeMatMul); // backward compatibility
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, TransposeMatMul); // backward compatibility
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, FusedMatMul);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, FusedMatMul);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, FusedMatMul);
|
||||
|
|
@ -84,7 +84,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1
|
|||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, FastGelu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, TransposeMatMul); // backward compatibility
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, TransposeMatMul); // backward compatibility
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, FusedMatMul);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, BFloat16_float, LayerNormalization);
|
||||
#endif
|
||||
|
|
@ -97,31 +97,31 @@ KernelCreateInfo BuildKernelCreateInfo<void>() {
|
|||
|
||||
Status RegisterCudaContribKernels(KernelRegistry& kernel_registry) {
|
||||
static const BuildKernelCreateInfoFn function_table[] = {
|
||||
BuildKernelCreateInfo<void>, //default entry to avoid the list become empty after ops-reducing
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, FastGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, FastGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, Gelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, Gelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, Gelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, BiasGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, BiasGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, BiasGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, TransposeMatMul)>, // backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, TransposeMatMul)>, // backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, TransposeMatMul)>, // backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, FusedMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, FusedMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, FusedMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, Rfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, Rfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, Rfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, Irfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, Irfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, Irfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, ComplexMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, ComplexMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, ComplexMulConj)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, ComplexMulConj)>,
|
||||
BuildKernelCreateInfo<void>, //default entry to avoid the list become empty after ops-reducing
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, FastGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, FastGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, Gelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, Gelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, Gelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, BiasGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, BiasGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, BiasGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, TransposeMatMul)>, // backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, TransposeMatMul)>, // backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, TransposeMatMul)>, // backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, FusedMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, FusedMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, FusedMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, Rfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, Rfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, Rfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, Irfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, Irfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, Irfft)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, ComplexMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, ComplexMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, ComplexMulConj)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, ComplexMulConj)>,
|
||||
|
||||
// These ops were experimental ops in onnx domain which have been removed now. We add them here as
|
||||
// contrib ops to maintain backward compatibility
|
||||
|
|
@ -164,21 +164,21 @@ Status RegisterCudaContribKernels(KernelRegistry& kernel_registry) {
|
|||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BiasSoftmax)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BiasDropout)>,
|
||||
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, int8_t_MLFloat16, QuantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, uint8_t_MLFloat16, QuantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, int8_t_MLFloat16, DequantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, uint8_t_MLFloat16, DequantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float_int8_t, QAttention)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16_int8_t, QAttention)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, int8_t_MLFloat16, QuantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, uint8_t_MLFloat16, QuantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, int8_t_MLFloat16, DequantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, uint8_t_MLFloat16, DequantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float_int8_t, QAttention)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16_int8_t, QAttention)>,
|
||||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, FastGelu)>,
|
||||
// TransposedMatMul is still here for backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, TransposeMatMul)>, // backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, FusedMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, BFloat16_float, LayerNormalization)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, FastGelu)>,
|
||||
// TransposedMatMul is still here for backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, TransposeMatMul)>, // backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, FusedMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, BFloat16_float, LayerNormalization)>,
|
||||
#endif
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, FusedConv)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, FusedConv)>,
|
||||
};
|
||||
|
||||
for (auto& function_table_entry : function_table) {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
#include "core/framework/kernel_registry.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace contrib {
|
||||
|
|
@ -12,4 +11,4 @@ Status RegisterCudaContribKernels(KernelRegistry& kernel_registry);
|
|||
|
||||
} // namespace cuda
|
||||
} // namespace contrib
|
||||
} // namespace onnxruntime
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -119,9 +119,9 @@ ONNX_OPERATOR_TYPED_KERNEL_EX(
|
|||
1,
|
||||
float,
|
||||
kCudaExecutionProvider,
|
||||
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
|
||||
(*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
|
||||
FusedConv<float>);
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace contrib
|
||||
} // namespace onnxruntime
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ ONNX_OPERATOR_KERNEL_EX(
|
|||
kMSDomain,
|
||||
1,
|
||||
kCudaExecutionProvider,
|
||||
KernelDefBuilder()
|
||||
(*KernelDefBuilder::Create())
|
||||
.TypeConstraint("T", BuildKernelDefConstraints<float, double, MLFloat16>()),
|
||||
Inverse);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/providers/shared_library/provider_api.h"
|
||||
#include "layer_norm.h"
|
||||
#include "layer_norm_impl.h"
|
||||
|
||||
#include "core/providers/common.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
@ -18,7 +17,7 @@ namespace cuda {
|
|||
1, \
|
||||
T##_##U, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()) \
|
||||
.TypeConstraint("U", DataTypeImpl::GetTensorType<U>()), \
|
||||
LayerNorm<T, U, false>); \
|
||||
|
|
@ -28,7 +27,7 @@ namespace cuda {
|
|||
1, \
|
||||
T##_##U, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()) \
|
||||
.TypeConstraint("U", DataTypeImpl::GetTensorType<U>()), \
|
||||
LayerNorm<T, U, true>);
|
||||
|
|
@ -59,7 +58,7 @@ Status LayerNorm<T, U, simplified>::ComputeInternal(OpKernelContext* ctx) const
|
|||
|
||||
auto X_data = reinterpret_cast<const CudaT*>(X->template Data<T>());
|
||||
auto scale_data = reinterpret_cast<const CudaT*>(scale->template Data<T>());
|
||||
auto bias_data = (simplified || (nullptr == bias)) ? nullptr: reinterpret_cast<const CudaT*>(bias->template Data<T>());
|
||||
auto bias_data = (simplified || (nullptr == bias)) ? nullptr : reinterpret_cast<const CudaT*>(bias->template Data<T>());
|
||||
|
||||
const TensorShape& x_shape = X->Shape();
|
||||
const int64_t axis = HandleNegativeAxis(axis_, x_shape.NumDimensions());
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
#include "core/common/common.h"
|
||||
#include "core/providers/cuda/cuda_kernel.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ ONNX_OPERATOR_KERNEL_EX(
|
|||
kMSDomain,
|
||||
1,
|
||||
kCudaExecutionProvider,
|
||||
KernelDefBuilder()
|
||||
(*KernelDefBuilder::Create())
|
||||
.TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES)
|
||||
.TypeConstraint("T1", ALL_IEEE_FLOAT_TENSOR_TYPES)
|
||||
.TypeConstraint("T2", DataTypeImpl::GetTensorType<bool>())
|
||||
.InputMemoryType<OrtMemTypeCPUInput>(3)
|
||||
.InputMemoryType<OrtMemTypeCPUInput>(4),
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 3)
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 4),
|
||||
BiasDropout);
|
||||
|
||||
template <typename T>
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "contrib_ops/cuda/math/bias_softmax.h"
|
||||
|
||||
#include "core/providers/common.h"
|
||||
|
||||
using namespace onnxruntime;
|
||||
using namespace onnxruntime::cuda;
|
||||
using namespace onnxruntime::contrib::cuda;
|
||||
|
|
@ -43,7 +42,7 @@ ONNX_OPERATOR_KERNEL_EX(
|
|||
kMSDomain,
|
||||
1,
|
||||
kCudaExecutionProvider,
|
||||
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::AllIEEEFloatTensorTypes()),
|
||||
(*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::AllIEEEFloatTensorTypes()),
|
||||
BiasSoftmax);
|
||||
|
||||
Status BiasSoftmax::ComputeInternal(OpKernelContext* ctx) const {
|
||||
|
|
@ -77,52 +76,52 @@ Status BiasSoftmax::ComputeInternal(OpKernelContext* ctx) const {
|
|||
|
||||
template <typename T>
|
||||
void DispatchBiasSoftmaxForward<T>::operator()(
|
||||
cudaStream_t stream,
|
||||
Tensor* output,
|
||||
const Tensor* input,
|
||||
const Tensor* input_bias,
|
||||
int element_count,
|
||||
int batch_count,
|
||||
int batch_stride,
|
||||
int bias_broadcast_size_per_batch) {
|
||||
DispatchBiasSoftmaxForwardImpl<T>(
|
||||
stream,
|
||||
output,
|
||||
input,
|
||||
input_bias,
|
||||
element_count,
|
||||
batch_count,
|
||||
batch_stride,
|
||||
bias_broadcast_size_per_batch);
|
||||
cudaStream_t stream,
|
||||
Tensor* output,
|
||||
const Tensor* input,
|
||||
const Tensor* input_bias,
|
||||
int element_count,
|
||||
int batch_count,
|
||||
int batch_stride,
|
||||
int bias_broadcast_size_per_batch) {
|
||||
DispatchBiasSoftmaxForwardImpl<T>(
|
||||
stream,
|
||||
output,
|
||||
input,
|
||||
input_bias,
|
||||
element_count,
|
||||
batch_count,
|
||||
batch_stride,
|
||||
bias_broadcast_size_per_batch);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void DispatchBiasSoftMaxForwardViaDnnLibrary<T>::operator()(
|
||||
cudaStream_t stream,
|
||||
cudnnHandle_t cudaDnnHandle,
|
||||
int element_count,
|
||||
int batch_count,
|
||||
int broadcast_axis,
|
||||
int softmax_axis,
|
||||
const onnxruntime::TensorShape& X_shape,
|
||||
const onnxruntime::Tensor* X,
|
||||
const onnxruntime::TensorShape& B_shape,
|
||||
const onnxruntime::Tensor* B,
|
||||
onnxruntime::Tensor* Y) {
|
||||
DispatchBiasSoftMaxForwardViaDnnLibraryImpl<T>(
|
||||
stream,
|
||||
cudaDnnHandle,
|
||||
element_count,
|
||||
batch_count,
|
||||
broadcast_axis,
|
||||
softmax_axis,
|
||||
X_shape,
|
||||
X,
|
||||
B_shape,
|
||||
B,
|
||||
Y);
|
||||
cudaStream_t stream,
|
||||
cudnnHandle_t cudaDnnHandle,
|
||||
int element_count,
|
||||
int batch_count,
|
||||
int broadcast_axis,
|
||||
int softmax_axis,
|
||||
const onnxruntime::TensorShape& X_shape,
|
||||
const onnxruntime::Tensor* X,
|
||||
const onnxruntime::TensorShape& B_shape,
|
||||
const onnxruntime::Tensor* B,
|
||||
onnxruntime::Tensor* Y) {
|
||||
DispatchBiasSoftMaxForwardViaDnnLibraryImpl<T>(
|
||||
stream,
|
||||
cudaDnnHandle,
|
||||
element_count,
|
||||
batch_count,
|
||||
broadcast_axis,
|
||||
softmax_axis,
|
||||
X_shape,
|
||||
X,
|
||||
B_shape,
|
||||
B,
|
||||
Y);
|
||||
}
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace contrib
|
||||
} // namespace onnxruntime
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
#include <limits>
|
||||
#include <algorithm>
|
||||
|
||||
#include "core/providers/common.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "core/providers/cuda/cudnn_common.h"
|
||||
#include "core/providers/cuda/cu_inc/binary_elementwise_impl.cuh"
|
||||
|
|
|
|||
|
|
@ -9,14 +9,14 @@ namespace onnxruntime {
|
|||
namespace contrib {
|
||||
namespace cuda {
|
||||
|
||||
#define CONTRIB_BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(x, ver, T) \
|
||||
ONNX_OPERATOR_TYPED_KERNEL_EX( \
|
||||
x, \
|
||||
kMSDomain, \
|
||||
ver, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
#define CONTRIB_BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED(x, ver, T) \
|
||||
ONNX_OPERATOR_TYPED_KERNEL_EX( \
|
||||
x, \
|
||||
kMSDomain, \
|
||||
ver, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
(*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
x<T>);
|
||||
|
||||
#define CONTRIB_BINARY_ELEMENTWISE_COMPUTE(x, T) \
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "complex_mul.h"
|
||||
#include "complex_mul_impl.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
@ -14,7 +13,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
ComplexMul<T, false>); \
|
||||
ONNX_OPERATOR_TYPED_KERNEL_EX( \
|
||||
|
|
@ -23,7 +22,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
ComplexMul<T, true>);
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ using namespace ::onnxruntime::cuda;
|
|||
template <typename T, bool is_conj>
|
||||
class ComplexMul : public BinaryElementwise<ShouldBroadcast> {
|
||||
public:
|
||||
ComplexMul(const OpKernelInfo info) : BinaryElementwise{info} {}
|
||||
ComplexMul(const OpKernelInfo& info) : BinaryElementwise{info} {}
|
||||
Status ComputeInternal(OpKernelContext* context) const override;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ void SetFFTState(FFTState* state,
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
Rfft<T>); \
|
||||
ONNX_OPERATOR_TYPED_KERNEL_EX( \
|
||||
|
|
@ -44,7 +44,7 @@ void SetFFTState(FFTState* state,
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
Irfft<T>);
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ namespace cuda {
|
|||
template <typename T>
|
||||
class FFTBase : public ::onnxruntime::cuda::CudaKernel {
|
||||
public:
|
||||
FFTBase(const OpKernelInfo info) : ::onnxruntime::cuda::CudaKernel{info}, normalized_{0}, onesided_{1} {
|
||||
FFTBase(const OpKernelInfo& info) : ::onnxruntime::cuda::CudaKernel{info}, normalized_{0}, onesided_{1} {
|
||||
ORT_ENFORCE((info.GetAttr("signal_ndim", &signal_ndim_)).IsOK(),
|
||||
"Attribute signal_ndim is missing in Node ", info.node().Name());
|
||||
ORT_ENFORCE(signal_ndim_ >= 1 && signal_ndim_ <= 3,
|
||||
|
|
@ -36,14 +36,14 @@ class FFTBase : public ::onnxruntime::cuda::CudaKernel {
|
|||
template <typename T>
|
||||
class Rfft final : public FFTBase<T> {
|
||||
public:
|
||||
Rfft(const OpKernelInfo info) : FFTBase<T>{info} {}
|
||||
Rfft(const OpKernelInfo& info) : FFTBase<T>{info} {}
|
||||
Status ComputeInternal(OpKernelContext* context) const override;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
class Irfft final : public FFTBase<T> {
|
||||
public:
|
||||
Irfft(const OpKernelInfo info) : FFTBase<T>{info} {}
|
||||
Irfft(const OpKernelInfo& info) : FFTBase<T>{info} {}
|
||||
Status ComputeInternal(OpKernelContext* context) const override;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
onnxruntime::cuda::MatMul<T>);
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("V", DataTypeImpl::GetTensorType<T>()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<bool>()), \
|
||||
IsAllFiniteOp<T>);
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
#include "attention_quantization.h"
|
||||
#include "attention_quantization_impl.cuh"
|
||||
#include "contrib_ops/cuda/bert/attention_impl.h"
|
||||
#include "core/framework/tensorprotoutils.h"
|
||||
#include "core/providers/common.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "core/providers/cuda/shared_inc/fpgeneric.h"
|
||||
#include "core/providers/cuda/shared_inc/integer_gemm.h"
|
||||
|
|
@ -25,11 +23,11 @@ namespace cuda {
|
|||
1, \
|
||||
T##_##TQuant, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
.InputMemoryType<OrtMemTypeCPUInput>(3) \
|
||||
.InputMemoryType<OrtMemTypeCPUInput>(4) \
|
||||
.InputMemoryType<OrtMemTypeCPUInput>(6) \
|
||||
.InputMemoryType<OrtMemTypeCPUInput>(7) \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 3) \
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 4) \
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 6) \
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 7) \
|
||||
.TypeConstraint("T1", DataTypeImpl::GetTensorType<TQuant>()) \
|
||||
.TypeConstraint("T2", DataTypeImpl::GetTensorType<TQuant>()) \
|
||||
.TypeConstraint("T3", DataTypeImpl::GetTensorType<T>()) \
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ using namespace onnxruntime::cuda;
|
|||
1, \
|
||||
T##_##U, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T1", DataTypeImpl::GetTensorType<U>()) \
|
||||
.TypeConstraint("T2", DataTypeImpl::GetTensorType<T>()), \
|
||||
QuantizeLinear<T, U>);
|
||||
|
|
@ -32,7 +32,7 @@ REGISTER_Q_KERNEL_TYPED(uint8_t, MLFloat16)
|
|||
1, \
|
||||
T##_##U, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T1", DataTypeImpl::GetTensorType<T>()) \
|
||||
.TypeConstraint("T2", DataTypeImpl::GetTensorType<U>()), \
|
||||
DequantizeLinear<T, U>);
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
Crop<T>);
|
||||
|
||||
|
|
|
|||
|
|
@ -4,21 +4,25 @@
|
|||
#include "core/providers/cuda/tensor/slice.h"
|
||||
#include "core/providers/cuda/tensor/slice_impl.h"
|
||||
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace contrib {
|
||||
namespace cuda {
|
||||
|
||||
using namespace onnxruntime::cuda;
|
||||
|
||||
#define REGISTER_TYPED_DYNAMICSLICE(TIND) \
|
||||
ONNX_OPERATOR_TYPED_KERNEL_EX( \
|
||||
DynamicSlice, \
|
||||
kOnnxDomain, \
|
||||
1, \
|
||||
TIND, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder().InputMemoryType<OrtMemTypeCPUInput>(1).InputMemoryType<OrtMemTypeCPUInput>(2).InputMemoryType<OrtMemTypeCPUInput>(3).TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()).TypeConstraint("Tind", DataTypeImpl::GetTensorType<TIND>()), \
|
||||
#define REGISTER_TYPED_DYNAMICSLICE(TIND) \
|
||||
ONNX_OPERATOR_TYPED_KERNEL_EX( \
|
||||
DynamicSlice, \
|
||||
kOnnxDomain, \
|
||||
1, \
|
||||
TIND, \
|
||||
kCudaExecutionProvider, \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 1) \
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 2) \
|
||||
.InputMemoryType(OrtMemTypeCPUInput, 3) \
|
||||
.TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()) \
|
||||
.TypeConstraint("Tind", DataTypeImpl::GetTensorType<TIND>()), \
|
||||
cuda::Slice<true>);
|
||||
|
||||
REGISTER_TYPED_DYNAMICSLICE(int32_t)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ namespace cuda {
|
|||
1, \
|
||||
T, \
|
||||
kCudaExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
(*KernelDefBuilder::Create()) \
|
||||
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
|
||||
ImageScaler<T>);
|
||||
|
||||
|
|
|
|||
|
|
@ -257,6 +257,20 @@ class PlannerImpl {
|
|||
|
||||
// Find if there exists some input tensor that we can use in-place for output_arg_num-th input in the node.
|
||||
bool FindReusableInput(const onnxruntime::Node& node, int output_arg_num, OrtValueIndex* reusable_input) {
|
||||
#ifdef ENABLE_TRAINING
|
||||
// Inputs of Yields are essentially the outputs for FW partial subgraph
|
||||
// Thses tensors will be pass back to pytorch, thus cannot share the buffer with other tensors
|
||||
|
||||
// Unhandled corner case:
|
||||
// If FW output tensor is consumed by BW graph, and pytorch performs an inplace operation on th returned tensor,
|
||||
// we will run into a buffer corruption problem.
|
||||
// One potential fix is returning a copy of output tensor, if it has downstream dependency
|
||||
auto p_next_node = node.OutputNodesBegin();
|
||||
if (p_next_node != node.OutputNodesEnd() && p_next_node->OpType() == "YieldOp") {
|
||||
return false;
|
||||
}
|
||||
#endif //ENABLE_TRAINING
|
||||
|
||||
auto p_output_arg = node.OutputDefs()[output_arg_num];
|
||||
const KernelCreateInfo& ci = GetKernelCreateInfo(kernel_create_info_map_, node.Index());
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@
|
|||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/framework/ml_value.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
class ExecutionProviders;
|
||||
|
|
|
|||
|
|
@ -80,8 +80,7 @@ void OrtValueTensorSlicer<T>::Iterator::MaterializeMLValue() const {
|
|||
//
|
||||
// TODO: Ideally we could avoid the overhead of creating a new Tensor (mainly cost of copying type and shape info)
|
||||
// and would simply update Tensor::p_data_ given all other info remains constant for each slice.
|
||||
auto sub_tensor = std::make_unique<Tensor>(tensor_data_type_, per_iteration_shape_,
|
||||
const_cast<void*>(tensor_slice_data_raw), *tensor_location_);
|
||||
auto sub_tensor = Tensor::Create(tensor_data_type_, per_iteration_shape_, const_cast<void*>(tensor_slice_data_raw), *tensor_location_);
|
||||
auto ml_tensor = DataTypeImpl::GetType<Tensor>();
|
||||
current_ = OrtValue{sub_tensor.release(), ml_tensor, ml_tensor->GetDeleteFunc()};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,9 +7,11 @@
|
|||
#include <limits>
|
||||
#include <type_traits>
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/ml_value.h"
|
||||
#include "core/framework/tensor.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
|
|||
|
|
@ -31,12 +31,12 @@ AllocatorPtr PrepackedWeightsContainer::GetOrCreateAllocator(const std::string&
|
|||
}
|
||||
|
||||
const PrePackedWeights& PrepackedWeightsContainer::GetWeight(const std::string& key) const {
|
||||
// .at() will throw if th key doesn't exist
|
||||
// .at() will throw if the key doesn't exist
|
||||
return prepacked_weights_map_.at(key);
|
||||
}
|
||||
|
||||
bool PrepackedWeightsContainer::WriteWeight(const std::string& key, PrePackedWeights&& packed_weight) {
|
||||
auto ret = prepacked_weights_map_.insert({key, std::move(packed_weight)});
|
||||
auto ret = prepacked_weights_map_.insert(std::make_pair(key, std::move(packed_weight)));
|
||||
return ret.second;
|
||||
}
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -6,5 +6,6 @@
|
|||
namespace onnxruntime {
|
||||
|
||||
bool InitProvidersSharedLibrary();
|
||||
void UnloadSharedProviders();
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include <vector>
|
||||
#include <type_traits>
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/common/path.h"
|
||||
#include "core/common/status.h"
|
||||
|
|
@ -26,15 +27,17 @@ bool operator==(const TensorShapeProto_Dimension& l, const TensorShapeProto_Dime
|
|||
bool operator!=(const TensorShapeProto_Dimension& l, const TensorShapeProto_Dimension& r);
|
||||
|
||||
} // namespace ONNX_NAMESPACE
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
class Tensor;
|
||||
namespace utils {
|
||||
#ifndef SHARED_PROVIDER
|
||||
|
||||
TensorShape GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShapeProto& tensor_shape_proto);
|
||||
|
||||
std::vector<int64_t> GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto);
|
||||
|
||||
/**
|
||||
/**
|
||||
* deserialize a TensorProto into a preallocated memory buffer.
|
||||
* \param tensor_proto_path A local file path of where the 'input' was loaded from. Can be NULL if the tensor proto doesn't
|
||||
* have any external data or it was loaded from current working dir. This path could be either a
|
||||
|
|
@ -51,8 +54,8 @@ common::Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_prot
|
|||
* @return
|
||||
*/
|
||||
common::Status TensorProtoToTensor(const Env& env, const ORTCHAR_T* model_path,
|
||||
const ONNX_NAMESPACE::TensorProto& tensor_proto,
|
||||
Tensor& tensor);
|
||||
const ONNX_NAMESPACE::TensorProto& tensor_proto,
|
||||
Tensor& tensor);
|
||||
|
||||
/** Creates a TensorProto from a Tensor.
|
||||
@param[in] tensor the Tensor whose data and shape will be used to create the TensorProto.
|
||||
|
|
@ -99,6 +102,7 @@ common::Status DenseTensorToSparseTensorProto(const ONNX_NAMESPACE::TensorProto&
|
|||
const Path& model_path,
|
||||
ONNX_NAMESPACE::SparseTensorProto& sparse);
|
||||
#endif // !ORT_MINIMAL_BUILD
|
||||
#endif
|
||||
|
||||
inline bool HasDimValue(const ONNX_NAMESPACE::TensorShapeProto_Dimension& dim) {
|
||||
return dim.value_case() == ONNX_NAMESPACE::TensorShapeProto_Dimension::kDimValue;
|
||||
|
|
@ -122,10 +126,12 @@ inline bool HasShape(const ONNX_NAMESPACE::TypeProto_Tensor& ten_proto) {
|
|||
return ten_proto.has_shape();
|
||||
}
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
inline bool HasShape(const ONNX_NAMESPACE::TypeProto_SparseTensor& ten_proto) {
|
||||
// XXX: Figure out how do in proto3
|
||||
return ten_proto.has_shape();
|
||||
}
|
||||
#endif
|
||||
|
||||
inline bool HasRawData(const ONNX_NAMESPACE::TensorProto& ten_proto) {
|
||||
// Can not be UNDEFINED and can not be STRING but test for STRING is usually performed separately
|
||||
|
|
@ -145,6 +151,7 @@ inline bool HasDataType(const ONNX_NAMESPACE::TensorProto& ten_proto) {
|
|||
return ten_proto.data_type() != ONNX_NAMESPACE::TensorProto::UNDEFINED;
|
||||
}
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
inline bool HasName(const ONNX_NAMESPACE::TensorProto& ten_proto) {
|
||||
return ten_proto.has_name(); // XXX
|
||||
}
|
||||
|
|
@ -168,11 +175,13 @@ inline bool HasKeyType(const ONNX_NAMESPACE::TypeProto_Map& map_proto) {
|
|||
inline bool HasValueType(const ONNX_NAMESPACE::TypeProto_Map& map_proto) {
|
||||
return map_proto.value_type().value_case() != ONNX_NAMESPACE::TypeProto::VALUE_NOT_SET;
|
||||
}
|
||||
#endif
|
||||
|
||||
inline bool HasType(const ONNX_NAMESPACE::ValueInfoProto& vi_proto) {
|
||||
return vi_proto.type().value_case() != ONNX_NAMESPACE::TypeProto::VALUE_NOT_SET;
|
||||
}
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
inline bool HasName(const ONNX_NAMESPACE::ValueInfoProto& vi_proto) {
|
||||
return vi_proto.has_name(); // XXX: Figure out proto3 way
|
||||
}
|
||||
|
|
@ -184,6 +193,7 @@ inline bool HasDomain(const ONNX_NAMESPACE::TypeProto_Opaque& op_proto) {
|
|||
inline bool HasName(const ONNX_NAMESPACE::TypeProto_Opaque& op_proto) {
|
||||
return !op_proto.name().empty();
|
||||
}
|
||||
#endif
|
||||
|
||||
inline bool HasType(const ONNX_NAMESPACE::AttributeProto& at_proto) {
|
||||
return at_proto.type() != ONNX_NAMESPACE::AttributeProto::AttributeType::AttributeProto_AttributeType_UNDEFINED;
|
||||
|
|
@ -229,6 +239,7 @@ inline bool HasGraphs(const ONNX_NAMESPACE::AttributeProto& at_proto) {
|
|||
return at_proto.type() == ONNX_NAMESPACE::AttributeProto::AttributeType::AttributeProto_AttributeType_GRAPHS;
|
||||
}
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
inline bool HasName(const ONNX_NAMESPACE::AttributeProto& at_proto) {
|
||||
return at_proto.has_name(); // XXX: Fugure out proto3
|
||||
}
|
||||
|
|
@ -249,6 +260,7 @@ inline bool HasName(const ONNX_NAMESPACE::NodeProto& node_proto) {
|
|||
//XXX: Figure out proto3 style
|
||||
return node_proto.has_name();
|
||||
}
|
||||
#endif
|
||||
|
||||
// UnpackTensor from raw data or the type specific data field. Does not handle external data.
|
||||
// If the tensor does not contain raw data then raw_data should be nullptr and raw_data_len should be 0.
|
||||
|
|
@ -276,6 +288,5 @@ common::Status UnpackInitializerData(const ONNX_NAMESPACE::TensorProto& initiali
|
|||
const Path& model_path,
|
||||
std::unique_ptr<unsigned char[]>& unpacked_tensor,
|
||||
size_t& tensor_byte_size) ORT_MUST_USE_RESULT;
|
||||
|
||||
} // namespace utils
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@
|
|||
#include "core/optimizer/matmul_scale_fusion.h"
|
||||
#include "core/optimizer/nchwc_transformer.h"
|
||||
#include "core/optimizer/nhwc_transformer.h"
|
||||
#include "core/optimizer/noop_elimination.h"
|
||||
#include "core/optimizer/not_where_fusion.h"
|
||||
#include "core/optimizer/relu_clip_fusion.h"
|
||||
#include "core/optimizer/reshape_fusion.h"
|
||||
|
|
@ -69,6 +70,7 @@ std::vector<std::unique_ptr<RewriteRule>> GenerateRewriteRules(
|
|||
rules.push_back(std::make_unique<EliminateDropout>());
|
||||
rules.push_back(std::make_unique<ExpandElimination>());
|
||||
rules.push_back(std::make_unique<CastElimination>());
|
||||
rules.push_back(std::make_unique<NoopElimination>());
|
||||
rules.push_back(std::make_unique<DivMulFusion>());
|
||||
rules.push_back(std::make_unique<FuseReluClip>());
|
||||
rules.push_back(std::make_unique<GemmTransposeFusion>());
|
||||
|
|
|
|||
96
onnxruntime/core/optimizer/noop_elimination.cc
Normal file
96
onnxruntime/core/optimizer/noop_elimination.cc
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/optimizer/noop_elimination.h"
|
||||
|
||||
#include "core/common/logging/logging.h"
|
||||
#include "core/graph/graph_viewer.h"
|
||||
#include "core/graph/op.h"
|
||||
#include "core/graph/graph_utils.h"
|
||||
#include "core/optimizer/initializer.h"
|
||||
#include "core/optimizer/rewrite_rule.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
/**
|
||||
Eliminate no op node - handling Add op for now
|
||||
Add example:
|
||||
|
||||
X 0
|
||||
\ /
|
||||
Add
|
||||
|
|
||||
Y
|
||||
*/
|
||||
Status NoopElimination::Apply(Graph& graph, Node& node, RewriteRuleEffect& rule_effect, const logging::Logger&) const {
|
||||
if (graph_utils::RemoveNode(graph, node)) {
|
||||
rule_effect = RewriteRuleEffect::kRemovedCurrentNode;
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
bool NoopElimination::SatisfyCondition(const Graph& graph, const Node& node, const logging::Logger& logger) const {
|
||||
|
||||
bool input0_is_initializer = graph_utils::IsConstantInitializer(graph, node.InputDefs()[0]->Name());
|
||||
bool input1_is_initializer = graph_utils::IsConstantInitializer(graph, node.InputDefs()[1]->Name());
|
||||
|
||||
// reject if both or neither inputs are initializers for now
|
||||
if (input0_is_initializer == input1_is_initializer) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto* initializer = graph_utils::GetConstantInitializer(graph, node.InputDefs()[input0_is_initializer ? 0 : 1]->Name());
|
||||
|
||||
// if initializer_rank is bigger, the output is expected to be initializer_rank per broadcasting rule,
|
||||
// but it won't happen if the case is accepted, thus reject it
|
||||
auto initializer_rank = initializer->dims().size();
|
||||
const auto* other_input_shape = node.InputDefs()[input0_is_initializer ? 1 : 0]->Shape();
|
||||
if (other_input_shape == nullptr || initializer_rank > other_input_shape->dim_size()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t data_type = initializer->data_type();
|
||||
Initializer add_init(*initializer, graph.ModelPath());
|
||||
if (add_init.size() > 1) {
|
||||
return false;
|
||||
}
|
||||
switch (data_type) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
|
||||
if (*add_init.data<float>() != 0.f) {
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16:
|
||||
if (math::halfToFloat(add_init.data<MLFloat16>()->val) != 0.f) {
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE:
|
||||
if (*add_init.data<double>() != static_cast<double>(0.f)) {
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT32:
|
||||
if (*add_init.data<int32_t>() != static_cast<int32_t>(0)) {
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_INT64:
|
||||
if (*add_init.data<int64_t>() != static_cast<int64_t>(0)) {
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
// reject node output is graph output for now
|
||||
if (!graph_utils::CanRemoveNode(graph, node, logger)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace onnxruntime
|
||||
31
onnxruntime/core/optimizer/noop_elimination.h
Normal file
31
onnxruntime/core/optimizer/noop_elimination.h
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "core/optimizer/rewrite_rule.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
/**
|
||||
@Class NoopElimination
|
||||
|
||||
Rewrite rule that eliminates the no op node.
|
||||
So far only Add node with 0 as one of its inputs is eliminated.
|
||||
But this class could be the placeholder for other no op nodes in future.
|
||||
*/
|
||||
class NoopElimination : public RewriteRule {
|
||||
public:
|
||||
NoopElimination() noexcept : RewriteRule("NoopElimination") {}
|
||||
|
||||
std::vector<std::string> TargetOpTypes() const noexcept override {
|
||||
return {"Add"};
|
||||
}
|
||||
|
||||
private:
|
||||
bool SatisfyCondition(const Graph& graph, const Node& node, const logging::Logger& logger) const override;
|
||||
|
||||
Status Apply(Graph& graph, Node& node, RewriteRuleEffect& rule_effect, const logging::Logger& logger) const override;
|
||||
}; // namespace onnxruntime
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -14,7 +14,11 @@ namespace onnxruntime {
|
|||
*/
|
||||
template <typename T>
|
||||
optional<T> ParseEnvironmentVariable(const std::string& name) {
|
||||
#ifndef SHARED_PROVIDER
|
||||
const std::string value_str = Env::Default().GetEnvironmentVar(name);
|
||||
#else
|
||||
const std::string value_str = GetEnvironmentVar(name);
|
||||
#endif
|
||||
if (value_str.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@
|
|||
#include <cstdint>
|
||||
#include <functional>
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/tensor.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
|
|||
|
|
@ -49,6 +49,8 @@ REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 13, 13, float);
|
|||
REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 13, 13, double);
|
||||
REGISTER_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 14, float);
|
||||
REGISTER_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 14, double);
|
||||
REGISTER_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 14, int8_t);
|
||||
REGISTER_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 14, int32_t);
|
||||
REGISTER_UNARY_ELEMENTWISE_KERNEL(Selu, 6);
|
||||
REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Sigmoid, 6, 12, float);
|
||||
REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Sigmoid, 6, 12, double);
|
||||
|
|
|
|||
|
|
@ -94,34 +94,24 @@ ONNX_CPU_OPERATOR_KERNEL(If,
|
|||
.TypeConstraint("V", DataTypeImpl::AllTensorAndSequenceTensorTypes()),
|
||||
If);
|
||||
|
||||
struct If::Info {
|
||||
Info(const onnxruntime::Node& node, const GraphViewer& subgraph_in) : subgraph(subgraph_in) {
|
||||
num_implicit_inputs = static_cast<int>(node.ImplicitInputDefs().size());
|
||||
used_implicit_inputs = std::vector<bool>(num_implicit_inputs, true);
|
||||
num_outputs = static_cast<int>(node.OutputDefs().size());
|
||||
If::Info::Info(const onnxruntime::Node& node, const GraphViewer& subgraph_in) : subgraph(subgraph_in) {
|
||||
num_implicit_inputs = static_cast<int>(node.ImplicitInputDefs().size());
|
||||
used_implicit_inputs = std::vector<bool>(num_implicit_inputs, true);
|
||||
num_outputs = static_cast<int>(node.OutputDefs().size());
|
||||
|
||||
auto& subgraph_outputs = subgraph.GetOutputs();
|
||||
auto num_subgraph_outputs = subgraph_outputs.size();
|
||||
auto& subgraph_outputs = subgraph.GetOutputs();
|
||||
auto num_subgraph_outputs = subgraph_outputs.size();
|
||||
|
||||
ORT_ENFORCE(num_subgraph_outputs == static_cast<size_t>(num_outputs),
|
||||
"'If' node has ", num_outputs, " outputs which doesn't match the subgraph's ",
|
||||
num_subgraph_outputs, " outputs.");
|
||||
ORT_ENFORCE(num_subgraph_outputs == static_cast<size_t>(num_outputs),
|
||||
"'If' node has ", num_outputs, " outputs which doesn't match the subgraph's ",
|
||||
num_subgraph_outputs, " outputs.");
|
||||
|
||||
subgraph_output_names.reserve(num_subgraph_outputs);
|
||||
for (size_t i = 0; i < num_subgraph_outputs; ++i) {
|
||||
auto& output = subgraph_outputs[i];
|
||||
subgraph_output_names.push_back(output->Name());
|
||||
}
|
||||
subgraph_output_names.reserve(num_subgraph_outputs);
|
||||
for (size_t i = 0; i < num_subgraph_outputs; ++i) {
|
||||
auto& output = subgraph_outputs[i];
|
||||
subgraph_output_names.push_back(output->Name());
|
||||
}
|
||||
|
||||
const GraphViewer& subgraph;
|
||||
|
||||
std::vector<bool> used_implicit_inputs;
|
||||
int num_implicit_inputs;
|
||||
int num_outputs;
|
||||
|
||||
std::vector<std::string> subgraph_output_names;
|
||||
};
|
||||
}
|
||||
|
||||
class IfImpl {
|
||||
public:
|
||||
|
|
@ -154,7 +144,7 @@ class IfImpl {
|
|||
std::vector<std::pair<AllocationType, OrtValue>> outputs_;
|
||||
};
|
||||
|
||||
If::If(const OpKernelInfo& info) : IControlFlowKernel(info) {
|
||||
void If::Init(const OpKernelInfo& info) {
|
||||
// make sure the required attributes are present even though we don't need it here.
|
||||
// The GraphProto attributes are loaded as a Graph instance by main Graph::Resolve,
|
||||
// and a SessionState instance for executing the subgraph is created by InferenceSession.
|
||||
|
|
@ -165,9 +155,6 @@ If::If(const OpKernelInfo& info) : IControlFlowKernel(info) {
|
|||
ORT_IGNORE_RETURN_VALUE(proto);
|
||||
}
|
||||
|
||||
// we need this to be in the .cc so 'unique_ptr<Info> info_' can be handled
|
||||
If::~If() = default;
|
||||
|
||||
common::Status If::SetupSubgraphExecutionInfo(const SessionState& session_state,
|
||||
const std::string& attribute_name,
|
||||
const SessionState& subgraph_session_state) {
|
||||
|
|
|
|||
|
|
@ -5,9 +5,6 @@
|
|||
#include <functional>
|
||||
#include "gsl/gsl"
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/feeds_fetches_manager.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#include "core/providers/cpu/controlflow/utils.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
@ -15,17 +12,25 @@ class SessionState;
|
|||
|
||||
class If : public controlflow::IControlFlowKernel {
|
||||
public:
|
||||
If(const OpKernelInfo& info);
|
||||
If(const OpKernelInfo& info) : IControlFlowKernel(info) { Init(info); }
|
||||
void Init(const OpKernelInfo& info);
|
||||
|
||||
Status Compute(OpKernelContext* ctx) const override;
|
||||
|
||||
common::Status SetupSubgraphExecutionInfo(const SessionState& session_state,
|
||||
const std::string& attribute_name,
|
||||
const SessionState& subgraph_session_state) override;
|
||||
Status SetupSubgraphExecutionInfo(const SessionState& session_state,
|
||||
const std::string& attribute_name,
|
||||
const SessionState& subgraph_session_state) override;
|
||||
|
||||
// hide internal implementation details via forward declaration.
|
||||
struct Info;
|
||||
~If();
|
||||
struct Info {
|
||||
Info(const onnxruntime::Node& node, const GraphViewer& subgraph_in);
|
||||
const GraphViewer& subgraph;
|
||||
|
||||
std::vector<bool> used_implicit_inputs;
|
||||
int num_implicit_inputs;
|
||||
int num_outputs;
|
||||
|
||||
std::vector<std::string> subgraph_output_names;
|
||||
};
|
||||
|
||||
private:
|
||||
// Info and FeedsFetchesManager re-used for each subgraph execution.
|
||||
|
|
|
|||
|
|
@ -122,56 +122,42 @@ ONNX_CPU_OPERATOR_KERNEL(Loop,
|
|||
.TypeConstraint("V", DataTypeImpl::AllTensorAndSequenceTensorTypes()),
|
||||
Loop);
|
||||
|
||||
struct Loop::Info {
|
||||
Info(const onnxruntime::Node& node, const GraphViewer& subgraph_in)
|
||||
: subgraph(subgraph_in) {
|
||||
num_loop_carried_vars = static_cast<int>(node.InputDefs().size()) - 2; // skip 'M' and 'cond'
|
||||
num_implicit_inputs = static_cast<int>(node.ImplicitInputDefs().size());
|
||||
num_subgraph_inputs = 2 + num_loop_carried_vars; // iter_num, cond, loop carried vars
|
||||
num_outputs = static_cast<int>(node.OutputDefs().size());
|
||||
Loop::Info::Info(const onnxruntime::Node& node, const GraphViewer& subgraph_in)
|
||||
: subgraph(subgraph_in) {
|
||||
num_loop_carried_vars = static_cast<int>(node.InputDefs().size()) - 2; // skip 'M' and 'cond'
|
||||
num_implicit_inputs = static_cast<int>(node.ImplicitInputDefs().size());
|
||||
num_subgraph_inputs = 2 + num_loop_carried_vars; // iter_num, cond, loop carried vars
|
||||
num_outputs = static_cast<int>(node.OutputDefs().size());
|
||||
|
||||
auto& subgraph_inputs = subgraph.GetInputs();
|
||||
auto& subgraph_outputs = subgraph.GetOutputs();
|
||||
auto& subgraph_inputs = subgraph.GetInputs();
|
||||
auto& subgraph_outputs = subgraph.GetOutputs();
|
||||
|
||||
// we know how many inputs we are going to call the subgraph with based on the Loop inputs,
|
||||
// and that value is in num_subgraph_inputs.
|
||||
// validate that the subgraph has that many inputs.
|
||||
ORT_ENFORCE(static_cast<size_t>(num_subgraph_inputs) == subgraph_inputs.size(),
|
||||
"Graph in 'body' attribute of Loop should have ", num_subgraph_inputs, " inputs. Found:",
|
||||
subgraph_inputs.size());
|
||||
// we know how many inputs we are going to call the subgraph with based on the Loop inputs,
|
||||
// and that value is in num_subgraph_inputs.
|
||||
// validate that the subgraph has that many inputs.
|
||||
ORT_ENFORCE(static_cast<size_t>(num_subgraph_inputs) == subgraph_inputs.size(),
|
||||
"Graph in 'body' attribute of Loop should have ", num_subgraph_inputs, " inputs. Found:",
|
||||
subgraph_inputs.size());
|
||||
|
||||
// check num outputs are correct. the 'cond' output from the subgraph is not a Loop output, so diff is 1
|
||||
num_subgraph_outputs = static_cast<int>(subgraph_outputs.size());
|
||||
ORT_ENFORCE(num_subgraph_outputs - 1 == num_outputs,
|
||||
"'Loop' node has ", num_outputs, " outputs so the subgraph requires ", num_outputs + 1,
|
||||
" but has ", num_subgraph_outputs);
|
||||
// check num outputs are correct. the 'cond' output from the subgraph is not a Loop output, so diff is 1
|
||||
num_subgraph_outputs = static_cast<int>(subgraph_outputs.size());
|
||||
ORT_ENFORCE(num_subgraph_outputs - 1 == num_outputs,
|
||||
"'Loop' node has ", num_outputs, " outputs so the subgraph requires ", num_outputs + 1,
|
||||
" but has ", num_subgraph_outputs);
|
||||
|
||||
subgraph_input_names.reserve(num_subgraph_inputs);
|
||||
for (int i = 0; i < num_subgraph_inputs; ++i) {
|
||||
subgraph_input_names.push_back(subgraph_inputs[i]->Name());
|
||||
}
|
||||
|
||||
// save list of subgraph output names in their provided order to use when fetching the results
|
||||
// from each subgraph execution. the Loop outputs will match this order.
|
||||
subgraph_output_names.reserve(num_subgraph_outputs);
|
||||
for (int i = 0; i < num_subgraph_outputs; ++i) {
|
||||
auto& output = subgraph_outputs[i];
|
||||
subgraph_output_names.push_back(output->Name());
|
||||
}
|
||||
subgraph_input_names.reserve(num_subgraph_inputs);
|
||||
for (int i = 0; i < num_subgraph_inputs; ++i) {
|
||||
subgraph_input_names.push_back(subgraph_inputs[i]->Name());
|
||||
}
|
||||
|
||||
const GraphViewer& subgraph;
|
||||
|
||||
int num_loop_carried_vars;
|
||||
int num_implicit_inputs;
|
||||
int num_outputs;
|
||||
|
||||
int num_subgraph_inputs;
|
||||
int num_subgraph_outputs;
|
||||
|
||||
std::vector<std::string> subgraph_input_names;
|
||||
std::vector<std::string> subgraph_output_names;
|
||||
};
|
||||
// save list of subgraph output names in their provided order to use when fetching the results
|
||||
// from each subgraph execution. the Loop outputs will match this order.
|
||||
subgraph_output_names.reserve(num_subgraph_outputs);
|
||||
for (int i = 0; i < num_subgraph_outputs; ++i) {
|
||||
auto& output = subgraph_outputs[i];
|
||||
subgraph_output_names.push_back(output->Name());
|
||||
}
|
||||
}
|
||||
|
||||
class LoopImpl {
|
||||
public:
|
||||
|
|
@ -246,7 +232,7 @@ static Status ConcatenateCpuOutput(void* /*stream*/,
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Loop::Loop(const OpKernelInfo& info) : IControlFlowKernel(info) {
|
||||
void Loop::Init(const OpKernelInfo& info) {
|
||||
// make sure the attribute was present even though we don't need it here.
|
||||
// The GraphProto is loaded as a Graph instance by main Graph::Resolve,
|
||||
// and a SessionState instance for executing the subgraph is created by InferenceSession.
|
||||
|
|
@ -259,8 +245,12 @@ Loop::Loop(const OpKernelInfo& info) : IControlFlowKernel(info) {
|
|||
stream_ = nullptr;
|
||||
}
|
||||
|
||||
// we need this to be in the .cc so 'unique_ptr<Info> info_' can be handled
|
||||
Loop::~Loop() = default;
|
||||
std::unique_ptr<OpKernel> Loop::Create(const OpKernelInfo& info, const ConcatOutput& concat_output_func, void* stream) {
|
||||
auto result = std::make_unique<Loop>(info);
|
||||
result->SetConcatOutputFunc(concat_output_func);
|
||||
result->SetComputeStream(stream);
|
||||
return result;
|
||||
}
|
||||
|
||||
common::Status Loop::SetupSubgraphExecutionInfo(const SessionState& session_state,
|
||||
const std::string& attribute_name,
|
||||
|
|
|
|||
|
|
@ -14,17 +14,30 @@ namespace onnxruntime {
|
|||
|
||||
class Loop : public controlflow::IControlFlowKernel {
|
||||
public:
|
||||
Loop(const OpKernelInfo& info);
|
||||
Loop(const OpKernelInfo& info) : IControlFlowKernel(info) { Init(info); }
|
||||
void Init(const OpKernelInfo& info);
|
||||
|
||||
Status Compute(OpKernelContext* ctx) const override;
|
||||
|
||||
common::Status SetupSubgraphExecutionInfo(const SessionState& session_state,
|
||||
const std::string& attribute_name,
|
||||
const SessionState& subgraph_session_state) override;
|
||||
Status SetupSubgraphExecutionInfo(const SessionState& session_state,
|
||||
const std::string& attribute_name,
|
||||
const SessionState& subgraph_session_state) override;
|
||||
|
||||
// hide internal implementation details via forward declaration.
|
||||
struct Info;
|
||||
~Loop();
|
||||
struct Info {
|
||||
Info(const onnxruntime::Node& node, const GraphViewer& subgraph_in);
|
||||
|
||||
const GraphViewer& subgraph;
|
||||
|
||||
int num_loop_carried_vars;
|
||||
int num_implicit_inputs;
|
||||
int num_outputs;
|
||||
|
||||
int num_subgraph_inputs;
|
||||
int num_subgraph_outputs;
|
||||
|
||||
std::vector<std::string> subgraph_input_names;
|
||||
std::vector<std::string> subgraph_output_names;
|
||||
};
|
||||
|
||||
// function to concatenate the OrtValue instances from each Loop iteration into a single output buffer.
|
||||
// @param per_iteration_output OrtValue instances from each iteration. Never empty. All should have the same shape.
|
||||
|
|
@ -32,6 +45,8 @@ class Loop : public controlflow::IControlFlowKernel {
|
|||
using ConcatOutput = std::function<Status(void* stream, std::vector<OrtValue>& per_iteration_output,
|
||||
void* output, size_t output_size_in_bytes)>;
|
||||
|
||||
static std::unique_ptr<OpKernel> Create(const OpKernelInfo& info, const ConcatOutput& concat_output_func, void* stream);
|
||||
|
||||
protected:
|
||||
// derived class can provide implementation for handling concatenation of Loop output on a different device
|
||||
void SetConcatOutputFunc(const ConcatOutput& concat_output_func) { concat_output_func_ = concat_output_func; }
|
||||
|
|
|
|||
|
|
@ -5,8 +5,11 @@
|
|||
#include <functional>
|
||||
#include "gsl/gsl"
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#endif
|
||||
|
||||
#include "core/framework/feeds_fetches_manager.h"
|
||||
#include "core/providers/cpu/controlflow/utils.h"
|
||||
#include "core/framework/ort_value_tensor_slicer.h"
|
||||
|
|
@ -14,6 +17,28 @@
|
|||
namespace onnxruntime {
|
||||
namespace scan {
|
||||
namespace detail {
|
||||
/**
|
||||
Helper struct for keeping static information about the Scan node and its subgraph.
|
||||
Used to create the FeedsFetchesManager needed for efficient subgraph execution.
|
||||
*/
|
||||
struct Info {
|
||||
Info(const Node& node, const GraphViewer& subgraph_in, int num_scan_inputs_in, bool is_v8);
|
||||
|
||||
const GraphViewer& subgraph;
|
||||
|
||||
int num_inputs;
|
||||
int num_variadic_inputs;
|
||||
int num_outputs;
|
||||
int num_loop_state_variables;
|
||||
int num_scan_inputs;
|
||||
int num_scan_outputs;
|
||||
|
||||
int num_implicit_inputs;
|
||||
|
||||
std::vector<std::string> subgraph_input_names;
|
||||
std::vector<std::string> subgraph_output_names;
|
||||
};
|
||||
|
||||
// helpers for handling data on a non-CPU device.
|
||||
// Provide as needed when Scan is being run by a non-CPU based ExecutionProvider
|
||||
struct DeviceHelpers {
|
||||
|
|
@ -44,19 +69,20 @@ struct DeviceHelpers {
|
|||
template <int OpSet>
|
||||
class Scan : public controlflow::IControlFlowKernel {
|
||||
public:
|
||||
Scan(const OpKernelInfo& info);
|
||||
Scan(const OpKernelInfo& info) : IControlFlowKernel(info) { Init(info); }
|
||||
void Init(const OpKernelInfo& info);
|
||||
|
||||
Status Compute(OpKernelContext* ctx) const override;
|
||||
|
||||
common::Status SetupSubgraphExecutionInfo(const SessionState& session_state,
|
||||
const std::string& attribute_name,
|
||||
const SessionState& subgraph_session_state) override;
|
||||
Status SetupSubgraphExecutionInfo(const SessionState& session_state,
|
||||
const std::string& attribute_name,
|
||||
const SessionState& subgraph_session_state) override;
|
||||
|
||||
// hide internal implementation details via forward declaration.
|
||||
struct Info;
|
||||
~Scan();
|
||||
struct Info : scan::detail::Info {
|
||||
Info(const onnxruntime::Node& node, const GraphViewer& subgraph_in, int num_scan_inputs_in)
|
||||
: scan::detail::Info(node, subgraph_in, num_scan_inputs_in, /* is_v8 */ OpSet == 8) {}
|
||||
};
|
||||
|
||||
protected:
|
||||
void SetDeviceHelpers(const scan::detail::DeviceHelpers& device_helpers) {
|
||||
device_helpers_ = device_helpers; // copy
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,12 +83,6 @@ ONNX_OPERATOR_SET_SCHEMA(
|
|||
.TypeConstraint("V", OpSchema::all_tensor_types(), "All Tensor types"));
|
||||
*/
|
||||
|
||||
template <>
|
||||
struct Scan<8>::Info : public scan::detail::Info {
|
||||
Info(const onnxruntime::Node& node, const GraphViewer& subgraph_in, int num_scan_inputs_in)
|
||||
: scan::detail::Info(node, subgraph_in, num_scan_inputs_in, /* is_v8 */ true) {}
|
||||
};
|
||||
|
||||
class Scan8Impl {
|
||||
public:
|
||||
Scan8Impl(OpKernelContextInternal& context,
|
||||
|
|
@ -135,7 +129,7 @@ class Scan8Impl {
|
|||
};
|
||||
|
||||
template <>
|
||||
Scan<8>::Scan(const OpKernelInfo& info) : IControlFlowKernel(info) {
|
||||
void Scan<8>::Init(const OpKernelInfo& info) {
|
||||
// make sure the attribute was present even though we don't need it here.
|
||||
// The GraphProto is loaded as a Graph instance by main Graph::Resolve,
|
||||
// and a SessionState instance for executing the subgraph is created by InferenceSession.
|
||||
|
|
@ -175,10 +169,6 @@ Status Scan<8>::SetupSubgraphExecutionInfo(const SessionState& session_state,
|
|||
return status;
|
||||
}
|
||||
|
||||
// we need this to be in the .cc so 'unique_ptr<Info> info_' can be handled
|
||||
template <>
|
||||
Scan<8>::~Scan() = default;
|
||||
|
||||
template <>
|
||||
Status Scan<8>::Compute(OpKernelContext* ctx) const {
|
||||
ORT_ENFORCE(feeds_fetches_manager_ && info_,
|
||||
|
|
|
|||
|
|
@ -97,12 +97,6 @@ ONNX_OPERATOR_SET_SCHEMA(
|
|||
}
|
||||
*/
|
||||
|
||||
template <>
|
||||
struct Scan<9>::Info : public scan::detail::Info {
|
||||
Info(const onnxruntime::Node& node, const GraphViewer& subgraph_in, int num_scan_inputs_in)
|
||||
: scan::detail::Info(node, subgraph_in, num_scan_inputs_in, /* is_v8 */ false) {}
|
||||
};
|
||||
|
||||
class ScanImpl {
|
||||
public:
|
||||
ScanImpl(OpKernelContextInternal& context,
|
||||
|
|
@ -159,7 +153,7 @@ class ScanImpl {
|
|||
};
|
||||
|
||||
template <>
|
||||
Scan<9>::Scan(const OpKernelInfo& info) : IControlFlowKernel(info) {
|
||||
void Scan<9>::Init(const OpKernelInfo& info) {
|
||||
// make sure the attribute was present even though we don't need it here.
|
||||
// The GraphProto is loaded as a Graph instance by main Graph::Resolve,
|
||||
// and a SessionState instance for executing the subgraph is created by InferenceSession.
|
||||
|
|
@ -202,10 +196,6 @@ Scan<9>::Scan(const OpKernelInfo& info) : IControlFlowKernel(info) {
|
|||
};
|
||||
}
|
||||
|
||||
// we need this to be in the .cc so 'unique_ptr<Info> info_' can be handled
|
||||
template <>
|
||||
Scan<9>::~Scan() = default;
|
||||
|
||||
template <>
|
||||
Status Scan<9>::SetupSubgraphExecutionInfo(const SessionState& session_state,
|
||||
const std::string& attribute_name,
|
||||
|
|
|
|||
|
|
@ -26,28 +26,6 @@ namespace detail {
|
|||
enum class ScanDirection { kForward = 0,
|
||||
kReverse = 1 };
|
||||
|
||||
/**
|
||||
Helper struct for keeping static information about the Scan node and its subgraph.
|
||||
Used to create the FeedsFetchesManager needed for efficient subgraph execution.
|
||||
*/
|
||||
struct Info {
|
||||
Info(const Node& node, const GraphViewer& subgraph_in, int num_scan_inputs_in, bool is_v8);
|
||||
|
||||
const GraphViewer& subgraph;
|
||||
|
||||
int num_inputs;
|
||||
int num_variadic_inputs;
|
||||
int num_outputs;
|
||||
int num_loop_state_variables;
|
||||
int num_scan_inputs;
|
||||
int num_scan_outputs;
|
||||
|
||||
int num_implicit_inputs;
|
||||
|
||||
std::vector<std::string> subgraph_input_names;
|
||||
std::vector<std::string> subgraph_output_names;
|
||||
};
|
||||
|
||||
/**
|
||||
Class to provide input/output OrtValue instances for a loop state variable.
|
||||
The OrtValue flips between two internal temporary buffers to minimize copies.
|
||||
|
|
|
|||
|
|
@ -7,12 +7,14 @@
|
|||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/feeds_fetches_manager.h"
|
||||
#ifdef SHARED_PROVIDER
|
||||
#include "core/framework/ml_value.h"
|
||||
#else
|
||||
#include "core/framework/op_kernel.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
class Graph;
|
||||
|
||||
// Creates a scalar MLValue based on given value and allocator.
|
||||
template <typename T>
|
||||
|
|
|
|||
|
|
@ -665,6 +665,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain,
|
|||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, int64_t, CumSum);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, float, Relu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, double, Relu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, int8_t, Relu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, int32_t, Relu);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, Trilu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, float, Add);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, double, Add);
|
||||
|
|
@ -1777,6 +1779,10 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
|
|||
Relu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, double,
|
||||
Relu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, int8_t,
|
||||
Relu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, int32_t,
|
||||
Relu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, Trilu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, float, Add)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, double, Add)>,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/common/type_list.h"
|
||||
#include "core/framework/data_types.h"
|
||||
|
|
@ -10,6 +11,7 @@
|
|||
#include "core/framework/op_kernel.h"
|
||||
#include "core/framework/tensorprotoutils.h"
|
||||
#include "core/providers/op_kernel_type_control_utils.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
@ -25,11 +27,17 @@ template <typename EnabledOutputTypeList = ConstantOfShapeDefaultOutputTypes>
|
|||
class ConstantOfShapeBase {
|
||||
protected:
|
||||
ConstantOfShapeBase(const OpKernelInfo& info) {
|
||||
#ifndef SHARED_PROVIDER
|
||||
ONNX_NAMESPACE::TensorProto t_proto;
|
||||
if (info.GetAttr<ONNX_NAMESPACE::TensorProto>("value", &t_proto).IsOK()) {
|
||||
ORT_ENFORCE(t_proto.dims_size() == 1, "Must have a single dimension");
|
||||
ORT_ENFORCE(t_proto.dims()[0] == 1, "Must have a single dimension of 1");
|
||||
SetValueFromTensorProto(t_proto);
|
||||
auto* t_proto_p = &t_proto;
|
||||
#else
|
||||
auto t_proto = ONNX_NAMESPACE::TensorProto::Create();
|
||||
auto* t_proto_p = t_proto.get();
|
||||
#endif
|
||||
if (info.GetAttr<ONNX_NAMESPACE::TensorProto>("value", t_proto_p).IsOK()) {
|
||||
ORT_ENFORCE(t_proto_p->dims_size() == 1, "Must have a single dimension");
|
||||
ORT_ENFORCE(t_proto_p->dims()[0] == 1, "Must have a single dimension of 1");
|
||||
SetValueFromTensorProto(*t_proto_p);
|
||||
} else {
|
||||
float f_value = 0.f;
|
||||
SetValue(sizeof(float), reinterpret_cast<void*>(&f_value));
|
||||
|
|
|
|||
|
|
@ -3,14 +3,16 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
namespace clip_internal {
|
||||
|
||||
template<typename T>
|
||||
template <typename T>
|
||||
class Clip_6Base {
|
||||
public:
|
||||
explicit Clip_6Base(const OpKernelInfo& info) {
|
||||
|
|
@ -20,11 +22,12 @@ class Clip_6Base {
|
|||
info.GetAttrOrDefault("max", &max_, max_val);
|
||||
ORT_ENFORCE(min_ <= max_);
|
||||
}
|
||||
|
||||
protected:
|
||||
T max_;
|
||||
T min_;
|
||||
};
|
||||
} // namespace clip_internal
|
||||
} // namespace clip_internal
|
||||
|
||||
template <typename T>
|
||||
class Clip_6 final : public clip_internal::Clip_6Base<T>, public OpKernel {
|
||||
|
|
@ -45,7 +48,7 @@ class Clip final : public OpKernel {
|
|||
Status Compute(OpKernelContext* ctx) const override;
|
||||
|
||||
private:
|
||||
template<typename T>
|
||||
template <typename T>
|
||||
struct ComputeImpl;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -3,10 +3,12 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#include "einsum_utils/einsum_compute_preprocessor.h"
|
||||
#include "einsum_utils/einsum_typed_compute_processor.h"
|
||||
#endif
|
||||
#include "einsum_utils/einsum_compute_preprocessor.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
|
|||
|
|
@ -49,10 +49,10 @@ Status MatMul(const T* input_1_data, const T* input_2_data, T* output_data,
|
|||
|
||||
// CPU specific ReduceSum helper
|
||||
template <typename T>
|
||||
Tensor ReduceSum(const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
concurrency::ThreadPool* tp, void* /*einsum_cuda_assets*/) {
|
||||
std::unique_ptr<Tensor> ReduceSum(const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
concurrency::ThreadPool* tp, void* /*einsum_cuda_assets*/) {
|
||||
return onnxruntime::ReduceSum<T>::Impl(input, reduce_axes,
|
||||
allocator, tp, keep_dims,
|
||||
input_shape_override);
|
||||
|
|
@ -351,8 +351,7 @@ std::unique_ptr<Tensor> ReduceSum(const Tensor& input, const std::vector<int64_t
|
|||
concurrency::ThreadPool* tp, void* einsum_cuda_assets,
|
||||
const DeviceHelpers::ReduceSum<T>& device_reduce_sum_func) {
|
||||
TensorShape overriden_shape(input_shape_override);
|
||||
auto output = device_reduce_sum_func(input, reduce_axes, true, allocator, &overriden_shape, tp, einsum_cuda_assets);
|
||||
return std::make_unique<Tensor>(std::move(output));
|
||||
return device_reduce_sum_func(input, reduce_axes, true, allocator, &overriden_shape, tp, einsum_cuda_assets);
|
||||
}
|
||||
|
||||
// Explicit template instantiations of functions
|
||||
|
|
@ -370,7 +369,7 @@ template std::unique_ptr<Tensor> MatMul<float>(
|
|||
AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets,
|
||||
const DeviceHelpers::MatMul<float>& device_matmul_func);
|
||||
|
||||
template Tensor DeviceHelpers::CpuDeviceHelpers::ReduceSum<float>(
|
||||
template std::unique_ptr<Tensor> DeviceHelpers::CpuDeviceHelpers::ReduceSum<float>(
|
||||
const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
|
|
@ -394,7 +393,7 @@ template std::unique_ptr<Tensor> MatMul<int32_t>(
|
|||
AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets,
|
||||
const DeviceHelpers::MatMul<int32_t>& device_matmul_func);
|
||||
|
||||
template Tensor DeviceHelpers::CpuDeviceHelpers::ReduceSum<int32_t>(
|
||||
template std::unique_ptr<Tensor> DeviceHelpers::CpuDeviceHelpers::ReduceSum<int32_t>(
|
||||
const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
|
|
@ -419,7 +418,7 @@ template std::unique_ptr<Tensor> MatMul<double>(
|
|||
AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets,
|
||||
const DeviceHelpers::MatMul<double>& device_matmul_func);
|
||||
|
||||
template Tensor DeviceHelpers::CpuDeviceHelpers::ReduceSum<double>(
|
||||
template std::unique_ptr<Tensor> DeviceHelpers::CpuDeviceHelpers::ReduceSum<double>(
|
||||
const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
|
|
@ -438,7 +437,7 @@ template Status DeviceHelpers::CpuDeviceHelpers::MatMul<int64_t>(
|
|||
size_t num_batches, size_t M, size_t K, size_t N, concurrency::ThreadPool* tp,
|
||||
void* einsum_cuda_assets);
|
||||
|
||||
template Tensor DeviceHelpers::CpuDeviceHelpers::ReduceSum<int64_t>(
|
||||
template std::unique_ptr<Tensor> DeviceHelpers::CpuDeviceHelpers::ReduceSum<int64_t>(
|
||||
const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
|
|
@ -470,3 +469,4 @@ template std::unique_ptr<Tensor> ReduceSum<MLFloat16>(
|
|||
|
||||
} // namespace EinsumOp
|
||||
} // namespace onnxruntime
|
||||
|
||||
|
|
|
|||
|
|
@ -6,9 +6,11 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/util/math.h"
|
||||
#include "core/providers/cpu/tensor/transpose.h"
|
||||
#include "core/providers/cpu/reduction/reduction_ops.h"
|
||||
#endif
|
||||
|
||||
#include <vector>
|
||||
|
||||
|
|
@ -39,10 +41,10 @@ using MatMul = std::function<Status(const T* input_1_data, const T* input_2_data
|
|||
|
||||
// ReduceSum op - Reduces along `reduce_axes`
|
||||
template <typename T>
|
||||
using ReduceSum = std::function<Tensor(const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
concurrency::ThreadPool* tp, void* einsum_cuda_assets)>;
|
||||
using ReduceSum = std::function<std::unique_ptr<Tensor>(const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
concurrency::ThreadPool* tp, void* einsum_cuda_assets)>;
|
||||
|
||||
// Diagonal op
|
||||
// Diagonal - A specialized implementation somewhat similar to Torch's Diagonal op
|
||||
|
|
@ -71,10 +73,10 @@ Status MatMul(const T* input_1_data, const T* input_2_data, T* output_data,
|
|||
void* einsum_cuda_assets);
|
||||
|
||||
template <typename T>
|
||||
Tensor ReduceSum(const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
concurrency::ThreadPool* tp, void* einsum_cuda_assets);
|
||||
std::unique_ptr<Tensor> ReduceSum(const Tensor& input, const std::vector<int64_t>& reduce_axes,
|
||||
bool keep_dims, AllocatorPtr allocator,
|
||||
const TensorShape* input_shape_override,
|
||||
concurrency::ThreadPool* tp, void* einsum_cuda_assets);
|
||||
|
||||
std::unique_ptr<Tensor> Diagonal(const Tensor& input, int64_t dim_1, int64_t dim_2, AllocatorPtr allocator, void* einsum_cuda_assets);
|
||||
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ struct EinsumEquationPreprocessor {
|
|||
// Subscript labels (letter) and subcript indices (a unique id to the letter) are interchangeable
|
||||
|
||||
// This is a pre-processor class that maps subscript labels to a dimension value, etc.
|
||||
#ifndef SHARED_PROVIDER
|
||||
class EinsumComputePreprocessor final {
|
||||
public:
|
||||
explicit EinsumComputePreprocessor(EinsumEquationPreprocessor& equation_preprocessor,
|
||||
|
|
@ -224,5 +225,5 @@ class EinsumComputePreprocessor final {
|
|||
// Holds EP-specific assets required for (auxiliary) ops that need to be executed on non-CPU EPs
|
||||
void* einsum_ep_assets_;
|
||||
};
|
||||
|
||||
#endif
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/tensor.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
template <typename T>
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/status.h"
|
||||
#include "core/framework/tensor.h"
|
||||
#endif
|
||||
#include <sstream>
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
|
|||
|
|
@ -3,16 +3,18 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/op_node_proto_helper.h"
|
||||
#include "core/providers/common.h"
|
||||
#include "core/util/math.h"
|
||||
#endif
|
||||
#include "core/framework/op_node_proto_helper.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
// A helper struct holding attributes for Conv-family ops
|
||||
struct ConvAttributes {
|
||||
explicit ConvAttributes(const OpNodeProtoHelper<ProtoHelperNodeContext>& info) {
|
||||
explicit ConvAttributes(const OpKernelInfo& info) {
|
||||
std::string auto_pad_str;
|
||||
auto status = info.GetAttr<std::string>("auto_pad", &auto_pad_str);
|
||||
if (status.IsOK()) {
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
namespace onnxruntime {
|
||||
|
||||
struct ConvTransposeAttributes : public ConvAttributes {
|
||||
explicit ConvTransposeAttributes(const OpNodeProtoHelper<ProtoHelperNodeContext>& info)
|
||||
explicit ConvTransposeAttributes(const OpKernelInfo& info)
|
||||
: ConvAttributes(info),
|
||||
output_padding(info.GetAttrsOrDefault<int64_t>("output_padding")),
|
||||
output_shape(info.GetAttrsOrDefault<int64_t>("output_shape")) {
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/status.h"
|
||||
#include "core/framework/tensor.h"
|
||||
#endif
|
||||
#include <sstream>
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@
|
|||
#pragma once
|
||||
|
||||
#include <cmath>
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/op_node_proto_helper.h"
|
||||
#include "core/framework/tensor_shape.h"
|
||||
#include "core/providers/common.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
@ -17,7 +19,13 @@ struct PoolAttributes {
|
|||
return op_name == "GlobalAveragePool" || op_name == "GlobalMaxPool" || op_name == "GlobalLpPool";
|
||||
}
|
||||
|
||||
#ifdef SHARED_PROVIDER
|
||||
// Shared providers don't know about OpNodeProtoHelper
|
||||
PoolAttributes(const OpKernelInfo& info,
|
||||
#else
|
||||
// Providers like Nuphar don't know about OpKernelInfo
|
||||
PoolAttributes(const OpNodeProtoHelper<ProtoHelperNodeContext>& info,
|
||||
#endif
|
||||
const std::string& op_name, int start_version)
|
||||
: global_pooling(IsGlobalPooling(op_name)) {
|
||||
if (global_pooling) {
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@
|
|||
#pragma once
|
||||
|
||||
#include <cmath>
|
||||
#ifndef SHARED_PROVIDER
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#include "core/providers/cpu/nn/pool_attributes.h"
|
||||
#include "core/util/math.h"
|
||||
#endif
|
||||
#include "core/providers/cpu/nn/pool_attributes.h"
|
||||
#include "core/mlas/inc/mlas.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
@ -106,8 +108,7 @@ class PoolBase {
|
|||
|
||||
protected:
|
||||
PoolBase(const OpKernelInfo& info)
|
||||
: op_name_(info.GetKernelDef().OpName().rfind("QLinear", 0) != 0 ?
|
||||
info.GetKernelDef().OpName() : info.GetKernelDef().OpName().substr(7)),
|
||||
: op_name_(info.GetKernelDef().OpName().rfind("QLinear", 0) != 0 ? info.GetKernelDef().OpName() : info.GetKernelDef().OpName().substr(7)),
|
||||
pool_attrs_(info, op_name_, GetStartVersion(info)) {
|
||||
}
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue