mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-30 23:18:20 +00:00
587e891cae
Implement CloudEP for hybrid inferencing. The PR introduces zero new API, customers could configure session and run options to do inferencing with Azure [triton endpoint.](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-deploy-with-triton?tabs=azure-cli%2Cendpoint) Sample configuration in python be like: ``` sess_opt.add_session_config_entry('cloud.endpoint_type', 'triton'); sess_opt.add_session_config_entry('cloud.uri', 'https://cloud.com'); sess_opt.add_session_config_entry('cloud.model_name', 'detection2'); sess_opt.add_session_config_entry('cloud.model_version', '7'); // optional, default 1 sess_opt.add_session_config_entry('cloud.verbose', '1'); // optional, default '0', meaning no verbose ... run_opt.add_run_config_entry('use_cloud', '1') # 0 for local inferencing, 1 for cloud endpoint. run_opt.add_run_config_entry('cloud.auth_key', '...') ... sess.run(None, {'input':input_}, run_opt) ``` Co-authored-by: Randy Shuai <rashuai@microsoft.com>
105 lines
No EOL
4 KiB
CMake
105 lines
No EOL
4 KiB
CMake
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
# Licensed under the MIT License.
|
|
|
|
include(ExternalProject)
|
|
|
|
if (WIN32)
|
|
|
|
function(get_vcpkg)
|
|
ExternalProject_Add(vcpkg
|
|
GIT_REPOSITORY https://github.com/microsoft/vcpkg.git
|
|
GIT_TAG 2022.11.14
|
|
PREFIX vcpkg
|
|
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/vcpkg-src
|
|
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/vcpkg-build
|
|
CONFIGURE_COMMAND ""
|
|
INSTALL_COMMAND ""
|
|
UPDATE_COMMAND ""
|
|
BUILD_COMMAND "<SOURCE_DIR>/bootstrap-vcpkg.bat")
|
|
|
|
ExternalProject_Get_Property(vcpkg SOURCE_DIR)
|
|
set(VCPKG_SRC ${SOURCE_DIR} PARENT_SCOPE)
|
|
set(VCPKG_DEPENDENCIES "vcpkg" PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
function(vcpkg_install PACKAGE_NAME)
|
|
add_custom_command(
|
|
OUTPUT ${VCPKG_SRC}/packages/${PACKAGE_NAME}_${onnxruntime_target_platform}-windows/BUILD_INFO
|
|
COMMAND ${VCPKG_SRC}/vcpkg install ${PACKAGE_NAME}:${onnxruntime_target_platform}-windows
|
|
WORKING_DIRECTORY ${VCPKG_SRC}
|
|
DEPENDS vcpkg)
|
|
|
|
add_custom_target(get${PACKAGE_NAME}
|
|
ALL
|
|
DEPENDS ${VCPKG_SRC}/packages/${PACKAGE_NAME}_${onnxruntime_target_platform}-windows/BUILD_INFO)
|
|
|
|
list(APPEND VCPKG_DEPENDENCIES "get${PACKAGE_NAME}")
|
|
set(VCPKG_DEPENDENCIES ${VCPKG_DEPENDENCIES} PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
get_vcpkg()
|
|
vcpkg_install(openssl)
|
|
vcpkg_install(openssl-windows)
|
|
vcpkg_install(rapidjson)
|
|
vcpkg_install(re2)
|
|
vcpkg_install(boost-interprocess)
|
|
vcpkg_install(boost-stacktrace)
|
|
vcpkg_install(zlib)
|
|
vcpkg_install(pthread)
|
|
vcpkg_install(b64)
|
|
|
|
add_dependencies(getb64 getpthread)
|
|
add_dependencies(getpthread getzlib)
|
|
add_dependencies(getzlib getboost-stacktrace)
|
|
add_dependencies(getboost-stacktrace getboost-interprocess)
|
|
add_dependencies(getboost-interprocess getre2)
|
|
add_dependencies(getre2 getrapidjson)
|
|
add_dependencies(getrapidjson getopenssl-windows)
|
|
add_dependencies(getopenssl-windows getopenssl)
|
|
|
|
ExternalProject_Add(triton
|
|
GIT_REPOSITORY https://github.com/triton-inference-server/client.git
|
|
GIT_TAG r22.12
|
|
PREFIX triton
|
|
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/triton-src
|
|
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/triton-build
|
|
CMAKE_ARGS -DVCPKG_TARGET_TRIPLET=${onnxruntime_target_platform}-windows -DCMAKE_TOOLCHAIN_FILE=${VCPKG_SRC}/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX=binary -DTRITON_ENABLE_CC_HTTP=ON
|
|
INSTALL_COMMAND ""
|
|
UPDATE_COMMAND "")
|
|
|
|
add_dependencies(triton ${VCPKG_DEPENDENCIES})
|
|
|
|
else()
|
|
|
|
ExternalProject_Add(rapidjson
|
|
GIT_REPOSITORY https://github.com/Tencent/rapidjson.git
|
|
GIT_TAG v1.1.0
|
|
PREFIX rapidjson
|
|
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/rapidjson-src
|
|
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/rapidjson-build
|
|
CMAKE_ARGS -DRAPIDJSON_BUILD_TESTS=OFF -DRAPIDJSON_BUILD_DOC=OFF -DRAPIDJSON_BUILD_EXAMPLES=OFF)
|
|
|
|
ExternalProject_Get_Property(rapidjson source_dir)
|
|
set(RAPIDJSON_INCLUDE_DIR ${source_dir}/include)
|
|
include_directories(${RAPIDJSON_INCLUDE_DIR})
|
|
|
|
ExternalProject_Add(triton
|
|
GIT_REPOSITORY https://github.com/triton-inference-server/client.git
|
|
GIT_TAG r22.12
|
|
PREFIX triton
|
|
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/triton-src
|
|
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps/triton-build
|
|
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=binary -DTRITON_ENABLE_CC_HTTP=ON
|
|
INSTALL_COMMAND ""
|
|
UPDATE_COMMAND "")
|
|
|
|
add_dependencies(triton rapidjson)
|
|
|
|
endif() #if (WIN32)
|
|
|
|
ExternalProject_Get_Property(triton SOURCE_DIR)
|
|
set(TRITON_SRC ${SOURCE_DIR})
|
|
|
|
ExternalProject_Get_Property(triton BINARY_DIR)
|
|
set(TRITON_BIN ${BINARY_DIR}/binary)
|
|
set(TRITON_THIRD_PARTY ${BINARY_DIR}/third-party) |