mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
Initial release of Vitis-AI Execution Provider (#3771)
* Initial release of Vitis-AI Execution Provider * Add documentation, fix for onnxruntime::Model changes and use stringstream instead of file dump for model passing * - Add Vitis-AI docker file - Add online quantization flow Vitis-AI execution provider - Fix remarks * - Add fatal error build message for Vitis-AI cmake build on Windows - Fix pep8 issue in build.py - Add Vitis-AI execution provider example in docs Co-authored-by: Elliott Delaye <elliott@xilinx.com> Co-authored-by: Jorn Tuyls <jornt@xilinx.com> Co-authored-by: Jorn Tuyls <jtuyls@users.noreply.github.com>
This commit is contained in:
edelaye
GitHub
parent
c42867c016
commit
64b5f7edf6
24 changed files with 953 additions and 6 deletions
15
BUILD.md
15
BUILD.md
|
|
@ -20,6 +20,7 @@
|
|||
* [DirectML](#DirectML)
|
||||
* [ARM Compute Library](#ARM-Compute-Library)
|
||||
* [Rockchip RKNPU](#RKNPU)
|
||||
* [Xilinx Vitis-AI](#Vitis-AI)
|
||||
* Options
|
||||
* [OpenMP](#OpenMP)
|
||||
* [OpenBLAS](#OpenBLAS)
|
||||
|
|
@ -517,8 +518,22 @@ See more information on the RKNPU Execution Provider [here](./docs/execution_pro
|
|||
onnxruntime_test_all
|
||||
rknpu_ddk/lib64/librknpu_ddk.so
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Vitis-AI
|
||||
See more information on the Xilinx Vitis-AI execution provider [here](./docs/execution_providers/Vitis-AI-ExecutionProvider.md).
|
||||
|
||||
For instructions to setup the hardware environment: [Hardware setup](./docs/execution_providers/Vitis-AI-ExecutionProvider.md#Hardware-setup)
|
||||
|
||||
#### Linux
|
||||
|
||||
```
|
||||
./build.sh --use_vitisai
|
||||
```
|
||||
#### Notes
|
||||
The Vitis-AI execution provider is only supported on Linux.
|
||||
|
||||
## Options
|
||||
### OpenMP
|
||||
#### Build Instructions
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ For production scenarios, it's strongly recommended to build only from an [offic
|
|||
|
||||
|CPU|GPU|IoT/Edge/Mobile|Other|
|
||||
|---|---|---|---|
|
||||
|<ul><li>Default CPU - *MLAS (Microsoft Linear Algebra Subprograms) + Eigen*</li><li>[Intel DNNL](./docs/execution_providers/DNNL-ExecutionProvider.md)</li><li>[Intel nGraph](./docs/execution_providers/nGraph-ExecutionProvider.md)</li><li>Intel MKL-ML *(build option)*</li></ul>|<ul><li>NVIDIA CUDA</li><li>[NVIDIA TensorRT](./docs/execution_providers/TensorRT-ExecutionProvider.md)</li><li>[DirectML](./docs/execution_providers/DirectML-ExecutionProvider.md)</li></ul>|<ul><li>[Intel OpenVINO](./docs/execution_providers/OpenVINO-ExecutionProvider.md)</li><li>[ARM Compute Library](./docs/execution_providers/ACL-ExecutionProvider.md) (*preview*)</li><li>[Android Neural Networks API](./docs/execution_providers/NNAPI-ExecutionProvider.md) (*preview*)</li></ul>|<ul><li>[Nuphar Model Compiler](./docs/execution_providers/Nuphar-ExecutionProvider.md) - (*preview*)</li><li>[Rockchip NPU](./docs/execution_providers/RKNPU-ExecutionProvider.md) (*preview*)</li></ul>|
|
||||
|<ul><li>Default CPU - *MLAS (Microsoft Linear Algebra Subprograms) + Eigen*</li><li>[Intel DNNL](./docs/execution_providers/DNNL-ExecutionProvider.md)</li><li>[Intel nGraph](./docs/execution_providers/nGraph-ExecutionProvider.md)</li><li>Intel MKL-ML *(build option)*</li></ul>|<ul><li>NVIDIA CUDA</li><li>[NVIDIA TensorRT](./docs/execution_providers/TensorRT-ExecutionProvider.md)</li><li>[DirectML](./docs/execution_providers/DirectML-ExecutionProvider.md)</li></ul>|<ul><li>[Intel OpenVINO](./docs/execution_providers/OpenVINO-ExecutionProvider.md)</li><li>[ARM Compute Library](./docs/execution_providers/ACL-ExecutionProvider.md) (*preview*)</li><li>[Android Neural Networks API](./docs/execution_providers/NNAPI-ExecutionProvider.md) (*preview*)</li></ul>|<ul><li>[Nuphar Model Compiler](./docs/execution_providers/Nuphar-ExecutionProvider.md) - (*preview*)</li><li>[Rockchip NPU](./docs/execution_providers/RKNPU-ExecutionProvider.md) (*preview*)</li><li>[Xilinx Vitis-AI](./docs/execution_providers/Vitis-AI-ExecutionProvider.md) (*preview*)</li></ul>|
|
||||
|
||||
* [Roadmap: Upcoming accelerators](./docs/Roadmap.md#accelerators-and-execution-providers)
|
||||
* [Extensibility: Add an execution provider](docs/AddingExecutionProvider.md)
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ option(onnxruntime_USE_OPENMP "Build with OpenMP support" OFF)
|
|||
option(onnxruntime_BUILD_SHARED_LIB "Build a shared library" OFF)
|
||||
option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable microsoft internal only code" OFF)
|
||||
option(onnxruntime_USE_NUPHAR "Build with Nuphar" OFF)
|
||||
option(onnxruntime_USE_VITISAI "Build with Vitis-AI" OFF)
|
||||
option(onnxruntime_USE_TENSORRT "Build with TensorRT support" OFF)
|
||||
option(onnxruntime_ENABLE_LTO "Enable link time optimization" OFF)
|
||||
option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF)
|
||||
|
|
@ -762,6 +763,17 @@ if(onnxruntime_USE_OPENVINO)
|
|||
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_VITISAI)
|
||||
if(WIN32)
|
||||
message(FATAL_ERROR "Vitis-AI execution provider is not supported on Windows.")
|
||||
else()
|
||||
add_definitions(-DUSE_VITISAI=1)
|
||||
include(pyxir)
|
||||
list(APPEND onnxruntime_EXTERNAL_LIBRARIES pyxir)
|
||||
list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES pyxir)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_OPENBLAS)
|
||||
add_definitions(-DUSE_OPENBLAS=1)
|
||||
if (WIN32)
|
||||
|
|
|
|||
28
cmake/external/pyxir.cmake
vendored
Normal file
28
cmake/external/pyxir.cmake
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
# Copyright(C) Xilinx Inc.
|
||||
# Licensed under the MIT License
|
||||
|
||||
set(PYXIR_SHARED_LIB libpyxir.so)
|
||||
|
||||
if(PYTHONINTERP_FOUND)
|
||||
execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
|
||||
"import pyxir as px; print(px.get_include_dir()); print(px.get_lib_dir());"
|
||||
RESULT_VARIABLE __result
|
||||
OUTPUT_VARIABLE __output
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
|
||||
if(__result MATCHES 0)
|
||||
string(REGEX REPLACE ";" "\\\\;" __values ${__output})
|
||||
string(REGEX REPLACE "\r?\n" ";" __values ${__values})
|
||||
list(GET __values 0 PYXIR_INCLUDE_DIR)
|
||||
list(GET __values 1 PYXIR_LIB_DIR)
|
||||
endif()
|
||||
else()
|
||||
message(STATUS "To find Pyxir, Python interpretater is required to be found.")
|
||||
endif()
|
||||
|
||||
add_library(pyxir SHARED IMPORTED)
|
||||
message("-- Found Pyxir lib: ${PYXIR_LIB_DIR}/${PYXIR_SHARED_LIB}")
|
||||
set_property(TARGET pyxir PROPERTY IMPORTED_LOCATION ${PYXIR_LIB_DIR}/${PYXIR_SHARED_LIB})
|
||||
|
||||
message("-- Found Pyxir include: ${PYXIR_INCLUDE_DIR}")
|
||||
include_directories(${PYXIR_INCLUDE_DIR})
|
||||
|
|
@ -84,6 +84,7 @@ target_link_libraries(onnxruntime PRIVATE
|
|||
${PROVIDERS_TENSORRT}
|
||||
${PROVIDERS_OPENVINO}
|
||||
${PROVIDERS_NUPHAR}
|
||||
${PROVIDERS_VITISAI}
|
||||
${PROVIDERS_DML}
|
||||
${PROVIDERS_ACL}
|
||||
${onnxruntime_winml}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,10 @@ if(onnxruntime_USE_NUPHAR)
|
|||
set(PROVIDERS_NUPHAR onnxruntime_providers_nuphar)
|
||||
list(APPEND ONNXRUNTIME_PROVIDER_NAMES nuphar)
|
||||
endif()
|
||||
if(onnxruntime_USE_VITISAI)
|
||||
set(PROVIDERS_VITISAI onnxruntime_providers_vitisai)
|
||||
list(APPEND ONNXRUNTIME_PROVIDER_NAMES vitisai)
|
||||
endif()
|
||||
if(onnxruntime_USE_CUDA)
|
||||
set(PROVIDERS_CUDA onnxruntime_providers_cuda)
|
||||
list(APPEND ONNXRUNTIME_PROVIDER_NAMES cuda)
|
||||
|
|
@ -413,6 +417,22 @@ if (onnxruntime_USE_NUPHAR)
|
|||
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/nuphar DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_VITISAI)
|
||||
file(GLOB_RECURSE onnxruntime_providers_vitisai_cc_srcs CONFIGURE_DEPENDS
|
||||
"${ONNXRUNTIME_ROOT}/core/providers/vitisai/*.h"
|
||||
"${ONNXRUNTIME_ROOT}/core/providers/vitisai/*.cc"
|
||||
)
|
||||
|
||||
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vitisai_cc_srcs})
|
||||
add_library(onnxruntime_providers_vitisai ${onnxruntime_providers_vitisai_cc_srcs})
|
||||
onnxruntime_add_include_to_target(onnxruntime_providers_vitisai onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf)
|
||||
add_dependencies(onnxruntime_providers_vitisai ${onnxruntime_EXTERNAL_DEPENDENCIES})
|
||||
set_target_properties(onnxruntime_providers_vitisai PROPERTIES FOLDER "ONNXRuntime")
|
||||
target_include_directories(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${VITISAI_INCLUDE_DIR})
|
||||
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/vitisai DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
|
||||
set_target_properties(onnxruntime_providers_vitisai PROPERTIES LINKER_LANGUAGE CXX)
|
||||
endif()
|
||||
|
||||
if (onnxruntime_USE_OPENVINO)
|
||||
|
||||
include_directories("${CMAKE_CURRENT_BINARY_DIR}/onnx")
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ set(onnxruntime_pybind11_state_libs
|
|||
${PROVIDERS_NGRAPH}
|
||||
${PROVIDERS_OPENVINO}
|
||||
${PROVIDERS_NUPHAR}
|
||||
${PROVIDERS_VITISAI}
|
||||
${PROVIDERS_NNAPI}
|
||||
${PROVIDERS_RKNPU}
|
||||
${PROVIDERS_DML}
|
||||
|
|
|
|||
37
dockerfiles/Dockerfile.vitisai
Normal file
37
dockerfiles/Dockerfile.vitisai
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# --------------------------------------------------------------
|
||||
# Copyright(C) Xilinx Inc.
|
||||
# Licensed under the MIT License.
|
||||
# --------------------------------------------------------------
|
||||
# Dockerfile to run ONNXRuntime with Vitis-AI integration
|
||||
|
||||
FROM xilinx/vitis-ai:latest
|
||||
|
||||
ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime
|
||||
ARG ONNXRUNTIME_BRANCH=master
|
||||
|
||||
ARG PYXIR_REPO=https://github.com/Xilinx/pyxir
|
||||
ARG PYXIR_BRANCH=master
|
||||
|
||||
RUN apt-get update &&\
|
||||
apt-get install -y sudo git bash
|
||||
|
||||
ENV PATH /code/cmake-3.14.3-Linux-x86_64/bin:$PATH
|
||||
ENV LD_LIBRARY_PATH /opt/xilinx/xrt/lib:$LD_LIBRARY_PATH
|
||||
|
||||
WORKDIR /code
|
||||
RUN . $VAI_ROOT/conda/etc/profile.d/conda.sh &&\
|
||||
conda activate vitis-ai-tensorflow &&\
|
||||
git clone --single-branch --branch ${PYXIR_BRANCH} --recursive ${PYXIR_REPO} pyxir &&\
|
||||
cd pyxir && python3 setup.py install --use_vai_rt
|
||||
RUN . $VAI_ROOT/conda/etc/profile.d/conda.sh &&\
|
||||
conda activate vitis-ai-tensorflow &&\
|
||||
git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime &&\
|
||||
/bin/sh onnxruntime/dockerfiles/scripts/install_common_deps.sh &&\
|
||||
cp onnxruntime/docs/Privacy.md /code/Privacy.md &&\
|
||||
cp onnxruntime/dockerfiles/LICENSE-IMAGE.txt /code/LICENSE-IMAGE.txt &&\
|
||||
cp onnxruntime/ThirdPartyNotices.txt /code/ThirdPartyNotices.txt &&\
|
||||
cd onnxruntime &&\
|
||||
/bin/sh ./build.sh --config RelWithDebInfo --enable_pybind --build_wheel --use_vitisai --parallel --update --build --build_shared_lib &&\
|
||||
pip install /code/onnxruntime/build/Linux/RelWithDebInfo/dist/*-linux_x86_64.whl &&\
|
||||
cd .. &&\
|
||||
rm -rf onnxruntime cmake-3.14.3-Linux-x86_64
|
||||
33
dockerfiles/scripts/docker_run_vitisai.sh
Executable file
33
dockerfiles/scripts/docker_run_vitisai.sh
Executable file
|
|
@ -0,0 +1,33 @@
|
|||
# --------------------------------------------------------------
|
||||
# Copyright(C) Xilinx Inc.
|
||||
# Licensed under the MIT License.
|
||||
# --------------------------------------------------------------
|
||||
|
||||
user=`whoami`
|
||||
uid=`id -u`
|
||||
gid=`id -g`
|
||||
|
||||
xclmgmt_driver="$(find /dev -name xclmgmt\*)"
|
||||
docker_devices=""
|
||||
for i in ${xclmgmt_driver} ;
|
||||
do
|
||||
docker_devices+="--device=$i "
|
||||
done
|
||||
|
||||
render_driver="$(find /dev/dri -name renderD\*)"
|
||||
for i in ${render_driver} ;
|
||||
do
|
||||
docker_devices+="--device=$i "
|
||||
done
|
||||
|
||||
docker run \
|
||||
$docker_devices \
|
||||
-v /opt/xilinx/dsa:/opt/xilinx/dsa \
|
||||
-v /opt/xilinx/overlaybins:/opt/xilinx/overlaybins \
|
||||
-e USER=$user -e UID=$uid -e GID=$gid \
|
||||
-v $PWD:/workspace \
|
||||
-w /workspace \
|
||||
-it \
|
||||
--rm \
|
||||
--network=host \
|
||||
onnxruntime-vitisai
|
||||
100
docs/execution_providers/Vitis-AI-ExecutionProvider.md
Normal file
100
docs/execution_providers/Vitis-AI-ExecutionProvider.md
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
<p align="center">
|
||||
<img src="./images/Vitis-AI.png">
|
||||
</p>
|
||||
|
||||
# Vitis-AI Execution Provider
|
||||
|
||||
[Vitis-AI](https://github.com/Xilinx/Vitis-AI) is Xilinx's development stack for hardware-accelerated AI inference on Xilinx platforms, including both edge devices and Alveo cards. It consists of optimized IP, tools, libraries, models, and example designs. It is designed with high efficiency and ease of use in mind, unleashing the full potential of AI acceleration on Xilinx FPGA and ACAP.
|
||||
|
||||
## Build
|
||||
|
||||
For build instructions, please see the [BUILD page](../../BUILD.md#Vitis-AI). Please setup the hardware environment before starting the build: [Hardware setup](#Hardware-setup).
|
||||
|
||||
### System requirements
|
||||
|
||||
The following table lists system requirements for running docker containers as well as Alveo cards.
|
||||
|
||||
|
||||
| **Component** | **Requirement** |
|
||||
|-----------------------------------------------------|------------------------------------------------------------|
|
||||
| Motherboard | PCI Express 3\.0\-compliant with one dual\-width x16 slot |
|
||||
| System Power Supply | 225W |
|
||||
| Operating System | Ubuntu 16\.04, 18\.04 |
|
||||
| | CentOS 7\.4, 7\.5 |
|
||||
| | RHEL 7\.4, 7\.5 |
|
||||
| CPU | Intel i3/i5/i7/i9/Xeon 64-bit CPU |
|
||||
| GPU \(Optional to accelerate quantization\) | NVIDIA GPU with a compute capability > 3.0 |
|
||||
| CUDA Driver \(Optional to accelerate quantization\) | nvidia\-410 |
|
||||
| FPGA | Xilinx Alveo U200 or U250 |
|
||||
| Docker Version | 19\.03\.1 |
|
||||
|
||||
### Hardware setup
|
||||
|
||||
1. Clone the Vitis AI repository:
|
||||
```
|
||||
git clone https://github.com/xilinx/vitis-ai
|
||||
```
|
||||
2. Install the Docker, and add the user to the docker group. Link the user to docker installation instructions from the following docker's website:
|
||||
* https://docs.docker.com/install/linux/docker-ce/ubuntu/
|
||||
* https://docs.docker.com/install/linux/docker-ce/centos/
|
||||
* https://docs.docker.com/install/linux/linux-postinstall/
|
||||
3. Any GPU instructions will have to be separated from Vitis AI.
|
||||
4. Set up Vitis AI to target Alveo cards. To target Alveo cards with Vitis AI for machine learning workloads, you must install the following software components:
|
||||
* Xilinx Runtime (XRT)
|
||||
* Alveo Deployment Shells (DSAs)
|
||||
* Xilinx Resource Manager (XRM) (xbutler)
|
||||
* Xilinx Overlaybins (Accelerators to Dynamically Load - binary programming files)
|
||||
|
||||
While it is possible to install all of these software components individually, a script has been provided to automatically install them at once. To do so:
|
||||
* Run the following commands:
|
||||
```
|
||||
cd Vitis-AI/alveo/packages
|
||||
sudo su
|
||||
./install.sh
|
||||
```
|
||||
* Power cycle the system.
|
||||
5. Build and start the ONNXRuntime Vitis-AI Docker Container.
|
||||
```
|
||||
cd {onnxruntime-root}/dockerfiles
|
||||
docker build -t onnxruntime-vitisai -f Dockerfile.vitisai .
|
||||
./scripts/docker_run_vitisai.sh
|
||||
```
|
||||
|
||||
Setup inside container
|
||||
```
|
||||
source /opt/xilinx/xrt/setup.sh
|
||||
conda activate vitis-ai-tensorflow
|
||||
```
|
||||
|
||||
|
||||
|
||||
### Samples
|
||||
|
||||
For python, you can base yourself on the following example:
|
||||
|
||||
```
|
||||
# Import pyxir before onnxruntime
|
||||
import pyxir
|
||||
import pyxir.frontend.onnx
|
||||
import pyxir.contrib.dpuv1.dpuv1
|
||||
|
||||
import onnxruntime
|
||||
|
||||
# Add other imports
|
||||
# ...
|
||||
|
||||
# Load inputs and do preprocessing
|
||||
# ...
|
||||
|
||||
# Create an inference session using the Vitis-AI execution provider
|
||||
session = onnxruntime.InferenceSession('[model_file].onnx', None,["VitisAIExecutionProvider"])
|
||||
|
||||
# First N (default = 128) inputs are used for quantization calibration and will
|
||||
# be executed on the CPU
|
||||
imput_name = [...]
|
||||
outputs = [session.run([], {input_name: calib_inputs[i]})[0] for i in range(128)]
|
||||
|
||||
# Afterwards, computations will be accelerated on the FPGA
|
||||
input_data = [...]
|
||||
result = session.run([], {input_name: input_data})
|
||||
```
|
||||
BIN
docs/execution_providers/images/Vitis-AI.png
Normal file
BIN
docs/execution_providers/images/Vitis-AI.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 328 KiB |
|
|
@ -22,12 +22,14 @@ constexpr const char* kMSNchwcDomain = "com.microsoft.nchwc";
|
|||
constexpr const char* kMSFeaturizersDomain = "com.microsoft.mlfeaturizers";
|
||||
constexpr const char* kMSDmlDomain = "com.microsoft.dml";
|
||||
constexpr const char* kNGraphDomain = "com.intel.ai";
|
||||
constexpr const char* kVitisAIDomain = "com.xilinx";
|
||||
constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider";
|
||||
constexpr const char* kCudaExecutionProvider = "CUDAExecutionProvider";
|
||||
constexpr const char* kDnnlExecutionProvider = "DnnlExecutionProvider";
|
||||
constexpr const char* kNGraphExecutionProvider = "NGRAPHExecutionProvider";
|
||||
constexpr const char* kOpenVINOExecutionProvider = "OpenVINOExecutionProvider";
|
||||
constexpr const char* kNupharExecutionProvider = "NupharExecutionProvider";
|
||||
constexpr const char* kVitisAIExecutionProvider = "VitisAIExecutionProvider";
|
||||
constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider";
|
||||
constexpr const char* kNnapiExecutionProvider = "NnapiExecutionProvider";
|
||||
constexpr const char* kRknpuExecutionProvider = "RknpuExecutionProvider";
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
// Copyright (c) Xilinx Inc.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "onnxruntime_c_api.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* \param use_arena zero: false. non-zero: true.
|
||||
*/
|
||||
ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_VITISAI, _In_ OrtSessionOptions* options, const char *backend_type, int device_id);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
@ -99,6 +99,7 @@ bool ProviderIsCpuBased(const std::string& provider_type) {
|
|||
provider_type == onnxruntime::kDnnlExecutionProvider ||
|
||||
provider_type == onnxruntime::kNGraphExecutionProvider ||
|
||||
provider_type == onnxruntime::kNupharExecutionProvider ||
|
||||
provider_type == onnxruntime::kVitisAIExecutionProvider ||
|
||||
provider_type == onnxruntime::kOpenVINOExecutionProvider ||
|
||||
provider_type == onnxruntime::kNnapiExecutionProvider ||
|
||||
provider_type == onnxruntime::kRknpuExecutionProvider;
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr
|
|||
provider != onnxruntime::kDnnlExecutionProvider &&
|
||||
provider != onnxruntime::kNGraphExecutionProvider &&
|
||||
provider != onnxruntime::kNupharExecutionProvider &&
|
||||
provider != onnxruntime::kVitisAIExecutionProvider &&
|
||||
provider != onnxruntime::kOpenVINOExecutionProvider &&
|
||||
provider != onnxruntime::kAclExecutionProvider) {
|
||||
TransformerMemcpyImpl copy_impl(graph, provider);
|
||||
|
|
@ -219,6 +220,7 @@ void TransformerMemcpyImpl::ProcessDefs(onnxruntime::Node& node, const KernelReg
|
|||
} else if (node_provider_type != kCudaExecutionProvider && node_provider_type != kTensorrtExecutionProvider) {
|
||||
// TODO: copy between devices? i.e. multiple GPUs
|
||||
if (node_provider_type != onnxruntime::kCpuExecutionProvider &&
|
||||
node_provider_type != onnxruntime::kVitisAIExecutionProvider &&
|
||||
node_provider_type != onnxruntime::kNGraphExecutionProvider && !node_provider_type.empty()) {
|
||||
ORT_THROW("Execution type '", node_provider_type, "' doesn't support memcpy ");
|
||||
}
|
||||
|
|
|
|||
1
onnxruntime/core/providers/vitisai/symbols.txt
Executable file
1
onnxruntime/core/providers/vitisai/symbols.txt
Executable file
|
|
@ -0,0 +1 @@
|
|||
OrtSessionOptionsAppendExecutionProvider_VITISAI
|
||||
170
onnxruntime/core/providers/vitisai/vitisai_custom_op.cc
Normal file
170
onnxruntime/core/providers/vitisai/vitisai_custom_op.cc
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
// Copyright(C) Xilinx Inc.
|
||||
// Licensed under the MIT License
|
||||
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable : 4244 4245)
|
||||
#elif __GNUC__
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#endif
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(default : 4244 4245)
|
||||
#elif __GNUC__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
#include <chrono>
|
||||
|
||||
#include <pyxir/common/xbuffer.hpp>
|
||||
#include <pyxir/frontend/onnx.hpp>
|
||||
#include <pyxir/runtime/run_options.hpp>
|
||||
|
||||
#include "vitisai_custom_op.h"
|
||||
#include "core/graph/graph_viewer.h"
|
||||
#include "core/graph/model.h"
|
||||
#include "core/common/logging/logging.h"
|
||||
#include "core/session/onnxruntime_cxx_api.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace vitisai_ep {
|
||||
|
||||
static ONNX_NAMESPACE::ModelProto GetModelProtoFromFusedNode(const onnxruntime::Node* fused_node,
|
||||
const logging::Logger& logger) {
|
||||
const auto* node_function = fused_node->GetFunctionBody();
|
||||
|
||||
ORT_ENFORCE(node_function != nullptr, "Could not extract function body for node: ",
|
||||
fused_node->Name());
|
||||
|
||||
const Graph& node_subgraph = node_function->Body();
|
||||
onnxruntime::Model model{node_subgraph.Name(), true, ModelMetaData{}, PathString{},
|
||||
IOnnxRuntimeOpSchemaRegistryList{}, node_subgraph.DomainToVersionMap(),
|
||||
std::vector<ONNX_NAMESPACE::FunctionProto>(), logger};
|
||||
|
||||
ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
|
||||
model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
|
||||
|
||||
*(model_proto.mutable_graph()) = node_subgraph.ToGraphProto();
|
||||
|
||||
return model_proto;
|
||||
}
|
||||
|
||||
VitisAICustomOp::VitisAICustomOp(const ComputeContext* context,
|
||||
const onnxruntime::Node* fused_node,
|
||||
const std::string &backend_type,
|
||||
const logging::Logger* logger)
|
||||
: backend_type_(backend_type)
|
||||
{
|
||||
SetLogger(logger);
|
||||
|
||||
allocate_func_ = context->allocate_func;
|
||||
release_func_ = context->release_func;
|
||||
allocator_ = context->allocator_handle;
|
||||
name_ = context->node_name;
|
||||
|
||||
model_proto_ = GetModelProtoFromFusedNode(fused_node, *GetLogger());
|
||||
|
||||
std::istringstream model_stream{model_proto_.SerializeAsString()};
|
||||
|
||||
xg_ = pyxir::onnx::import_onnx_model(model_stream);
|
||||
pyxir::partition(xg_, std::vector<std::string>{backend_type_}, "");
|
||||
|
||||
auto input_defs = fused_node->InputDefs();
|
||||
for (auto idef : input_defs) {
|
||||
// std::cout << "DPU input def: " << idef->Name() << std::endl;
|
||||
in_tensor_names_.push_back(idef->Name());
|
||||
}
|
||||
|
||||
auto output_defs = fused_node->OutputDefs();
|
||||
for (auto odef : output_defs) {
|
||||
// std::cout << "DPU output def: " << odef->Name() << std::endl;
|
||||
out_tensor_names_.push_back(odef->Name());
|
||||
}
|
||||
|
||||
pyxir::RunOptionsHolder run_options(new pyxir::runtime::RunOptions());
|
||||
run_options->online_quantization = true;
|
||||
rt_mod_ = pyxir::build_rt(xg_, backend_type_, in_tensor_names_, out_tensor_names_,
|
||||
"vai", run_options);
|
||||
}
|
||||
|
||||
VitisAICustomOp::~VitisAICustomOp() {}
|
||||
|
||||
|
||||
Status VitisAICustomOp::Compute(const OrtApi* api, OrtKernelContext* context) const {
|
||||
Ort::CustomOpApi ort{*api};
|
||||
const unsigned num_inputs = (unsigned) xg_->get_nb_inputs();
|
||||
|
||||
ssize_t batch_size = 1;
|
||||
std::vector<pyxir::XBufferHolder> in_tensors;
|
||||
std::vector<pyxir::XBufferHolder> out_tensors;
|
||||
|
||||
// Initialize input tensors.
|
||||
try {
|
||||
for (unsigned i = 0; i < num_inputs; ++i) {
|
||||
// std::cout << "Input name: " << in_tensor_names_[i];
|
||||
const OrtValue* input_tensor = ort.KernelContext_GetInput(context, i);
|
||||
auto tensor_info = ort.GetTensorTypeAndShape(input_tensor);
|
||||
auto tensor_type = ort.GetTensorElementType(tensor_info);
|
||||
auto ort_shape = ort.GetTensorShape(tensor_info);
|
||||
std::vector<ssize_t> tensor_shape{ort_shape.begin(), ort_shape.end()};
|
||||
batch_size = tensor_shape[0];
|
||||
|
||||
if (tensor_type != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
|
||||
"VITIS-AI EP input onnx tensor data type: " + std::to_string(tensor_type) + " not supported.");
|
||||
|
||||
void* input_data = const_cast<void*>(ort.GetTensorData<void>(input_tensor));
|
||||
in_tensors.push_back(std::shared_ptr<pyxir::XBuffer>(
|
||||
new pyxir::XBuffer(input_data, 4, "f", tensor_shape.size(), tensor_shape,
|
||||
false, false)));
|
||||
}
|
||||
} catch (const std::exception& exp) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Exception while copying input data to Pyxir: " + std::string(exp.what()));
|
||||
} catch (...) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Unknown exception while copying input data to Pyxir");
|
||||
}
|
||||
|
||||
// Initialize output tensors
|
||||
try {
|
||||
for (unsigned i = 0; i < out_tensor_names_.size(); ++i) {
|
||||
auto shape = xg_->get(out_tensor_names_[i])->shapes[0];
|
||||
std::vector<ssize_t> out_shape{shape.begin(), shape.end()};
|
||||
out_shape[0] = batch_size;
|
||||
std::vector<int64_t> ort_shape{out_shape.begin(), out_shape.end()};
|
||||
|
||||
OrtValue* output_tensor = ort.KernelContext_GetOutput(context, i, ort_shape.data(), ort_shape.size());
|
||||
auto tensor_info = ort.GetTensorTypeAndShape(output_tensor);
|
||||
auto tensor_type = ort.GetTensorElementType(tensor_info);
|
||||
if (tensor_type != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
|
||||
"VITIS-AI EP input onnx tensor data type: " + std::to_string(tensor_type) + " not supported.");
|
||||
|
||||
void* output_data = ort.GetTensorMutableData<void>(output_tensor);
|
||||
out_tensors.push_back(std::shared_ptr<pyxir::XBuffer>(
|
||||
new pyxir::XBuffer(output_data, 4, "f", out_shape.size(), out_shape,
|
||||
false, false)));
|
||||
}
|
||||
} catch (const std::exception& exp) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Exception while creating Pyxir output Tensor: " + std::string(exp.what()));
|
||||
} catch (...) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Unknown exception while creating Pyxir output Tensor");
|
||||
}
|
||||
|
||||
// Run the graph through Vitis-AI Pyxir
|
||||
try {
|
||||
// std::lock_guard<std::mutex> lock(compute_lock_);
|
||||
rt_mod_->execute(in_tensors, out_tensors);
|
||||
} catch (const std::exception& exp) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Exception while executing Pyxir computation: " + std::string(exp.what()));
|
||||
} catch (...) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Unknown exception while executing Pyxir computation");
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace vitisai_ep
|
||||
} // namespace onnxruntime
|
||||
74
onnxruntime/core/providers/vitisai/vitisai_custom_op.h
Normal file
74
onnxruntime/core/providers/vitisai/vitisai_custom_op.h
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
// Copyright(C) Xilinx Inc.
|
||||
// Licensed under the MIT License
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable : 4244 4245)
|
||||
#elif __GNUC__
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#endif
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(default : 4244 4245)
|
||||
#elif __GNUC__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
#include <pyxir/pyxir.hpp>
|
||||
|
||||
#include "core/session/onnxruntime_c_api.h"
|
||||
#include "core/framework/func_api.h"
|
||||
#include "core/graph/graph_viewer.h"
|
||||
#include "core/common/logging/logging.h"
|
||||
#include "core/graph/onnx_protobuf.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace vitisai_ep {
|
||||
|
||||
class VitisAICustomOp {
|
||||
public:
|
||||
VitisAICustomOp(const ComputeContext* context,
|
||||
const onnxruntime::Node* fused_node,
|
||||
const std::string &backend_type,
|
||||
const logging::Logger* logger);
|
||||
|
||||
Status Compute(const OrtApi* api, OrtKernelContext* context) const;
|
||||
|
||||
~VitisAICustomOp();
|
||||
|
||||
void SetLogger(const logging::Logger* logger) {
|
||||
logger_ = logger;
|
||||
}
|
||||
|
||||
const logging::Logger* GetLogger() const {
|
||||
return logger_;
|
||||
}
|
||||
|
||||
private:
|
||||
Status Initialize(const OrtApi* api, OrtKernelContext* context) const;
|
||||
|
||||
std::vector<std::string> in_tensor_names_;
|
||||
std::vector<std::string> out_tensor_names_;
|
||||
pyxir::XGraphHolder xg_;
|
||||
|
||||
std::string backend_type_;
|
||||
|
||||
pyxir::RtModHolder rt_mod_ = nullptr;
|
||||
|
||||
AllocateFunc allocate_func_ = nullptr;
|
||||
|
||||
DestroyFunc release_func_ = nullptr;
|
||||
|
||||
AllocatorHandle allocator_ = nullptr;
|
||||
|
||||
std::string name_;
|
||||
|
||||
mutable std::mutex compute_lock_;
|
||||
|
||||
const logging::Logger* logger_ = nullptr;
|
||||
|
||||
ONNX_NAMESPACE::ModelProto model_proto_;
|
||||
};
|
||||
} // namespace vitisai_ep
|
||||
} // namespace onnxruntime
|
||||
324
onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc
Normal file
324
onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
// Copyright (c) Xilinx Inc. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include <istream>
|
||||
#include <fstream>
|
||||
|
||||
#include <pyxir/pyxir.hpp>
|
||||
#include <pyxir/frontend/onnx.hpp>
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/common/logging/logging.h"
|
||||
#include "core/framework/compute_capability.h"
|
||||
#include "core/framework/allocatormgr.h"
|
||||
#include "core/framework/kernel_registry.h"
|
||||
#include "core/graph/graph_viewer.h"
|
||||
#include "core/graph/model.h"
|
||||
#include "vitisai_execution_provider.h"
|
||||
#include "vitisai_custom_op.h"
|
||||
|
||||
#define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, std::min(destsz, srcsz))
|
||||
|
||||
using namespace ONNX_NAMESPACE;
|
||||
using namespace ::onnxruntime::logging;
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
constexpr const char* PREFIX = "VITISAI";
|
||||
|
||||
typedef std::shared_ptr<pyxir::graph::XGraph> XGraphHolder;
|
||||
typedef std::shared_ptr<pyxir::graph::XLayer> XLayerHolder;
|
||||
|
||||
VitisAIExecutionProvider::VitisAIExecutionProvider(const VitisAIExecutionProviderInfo& info)
|
||||
: IExecutionProvider{onnxruntime::kVitisAIExecutionProvider}, backend_type_(info.backend_type),
|
||||
device_id_(info.device_id) {
|
||||
|
||||
auto default_allocator_factory = [](int) {
|
||||
auto memory_info = onnxruntime::make_unique<OrtMemoryInfo>(PREFIX, OrtAllocatorType::OrtDeviceAllocator);
|
||||
return onnxruntime::make_unique<CPUAllocator>(std::move(memory_info));
|
||||
};
|
||||
|
||||
DeviceAllocatorRegistrationInfo default_memory_info{
|
||||
OrtMemTypeDefault,
|
||||
std::move(default_allocator_factory),
|
||||
std::numeric_limits<size_t>::max()
|
||||
};
|
||||
|
||||
InsertAllocator(CreateAllocator(default_memory_info));
|
||||
}
|
||||
|
||||
|
||||
static std::vector<NodeIndex>
|
||||
GetUnsupportedNodeIndices(const XGraphHolder &xg, const std::string &backend_type,
|
||||
const GraphViewer& graph_viewer,
|
||||
/*out*/ std::unordered_set<std::string>& required_initializers) {
|
||||
|
||||
// Retrieve
|
||||
std::set<std::string> supported_tensors;
|
||||
for (auto &xl_name : xg->get_layer_names()) {
|
||||
XLayerHolder xl = xg->get(xl_name);
|
||||
if (xl->target == backend_type)
|
||||
supported_tensors.insert(xl->get_attr("onnx_id").get_string());
|
||||
}
|
||||
|
||||
std::vector<NodeIndex> unsupported_nodes_idx;
|
||||
|
||||
for (const auto& node_idx : graph_viewer.GetNodesInTopologicalOrder()) {
|
||||
ConstPointerContainer<std::vector<NodeArg*>> node_args
|
||||
= graph_viewer.GetNode(node_idx)->OutputDefs();
|
||||
|
||||
bool is_node_supported = false;
|
||||
for (ConstPointerContainer<std::vector<NodeArg*>>::ConstIterator it =
|
||||
node_args.begin(); it != node_args.end(); ++it) {
|
||||
if (supported_tensors.find((*it)->Name()) != supported_tensors.end()) {
|
||||
is_node_supported = true;
|
||||
} else if (is_node_supported) {
|
||||
// Some output tensors are supported but not others,
|
||||
// should not happen
|
||||
LOGS_DEFAULT(FATAL) << "VITIS-AI EP: Found node output tensor "
|
||||
<< (*it)->Name() << " which is partially supported by "
|
||||
<< " DPU accelerator. This is an invalid case";
|
||||
}
|
||||
}
|
||||
|
||||
if (is_node_supported) {
|
||||
// Collect inputs that are initializers
|
||||
graph_viewer.GetNode(node_idx)->ForEachDef([&required_initializers, &graph_viewer](const onnxruntime::NodeArg& node_arg, bool is_input) {
|
||||
if(is_input && graph_viewer.GetAllInitializedTensors().count(node_arg.Name())) {
|
||||
required_initializers.insert(node_arg.Name());
|
||||
} }, true);
|
||||
} else {
|
||||
unsupported_nodes_idx.push_back(node_idx);
|
||||
}
|
||||
}
|
||||
|
||||
return unsupported_nodes_idx;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a vector clusters(or node_idx). For each unsupported node, the graph is split into 3 parts.
|
||||
* supported_cluster + (UNsupported_node + rest_of_the_graph). This functions returns vector of all supported_clusters by DPU
|
||||
*/
|
||||
static std::vector<std::vector<NodeIndex>>
|
||||
GetPartitionedClusters(const std::vector<NodeIndex>& topological_order, const std::vector<NodeIndex>& unsupported_nodes) {
|
||||
std::vector<std::vector<NodeIndex>> clusters;
|
||||
|
||||
auto prev = topological_order.begin();
|
||||
|
||||
for (const auto& unsup_node : unsupported_nodes) {
|
||||
auto it = std::find(prev, topological_order.end(), unsup_node);
|
||||
// Create a cluster vector[supported_node_idx, unsupported_node_idx) and append it to return list.
|
||||
std::vector<NodeIndex> this_cluster{prev, it};
|
||||
if (!this_cluster.empty()) {
|
||||
clusters.push_back(std::move(this_cluster));
|
||||
}
|
||||
// Point prev to node idx past this unsuported node.
|
||||
prev = ++it;
|
||||
}
|
||||
|
||||
//Tail
|
||||
std::vector<NodeIndex> this_cluster{prev, topological_order.end()};
|
||||
if (!this_cluster.empty()) {
|
||||
clusters.push_back(std::move(this_cluster));
|
||||
}
|
||||
|
||||
return clusters;
|
||||
}
|
||||
|
||||
static void GetInputsOutputsOfCluster(const GraphViewer& graph_viewer,
|
||||
const std::vector<NodeIndex>& cluster,
|
||||
const std::unordered_set<std::string>& ng_required_initializers,
|
||||
/*out*/ std::vector<std::string>& cluster_inputs,
|
||||
/*out*/ std::vector<std::string>& cluster_outputs) {
|
||||
std::unordered_set<std::string> input_args;
|
||||
std::vector<std::string> ordered_input_args;
|
||||
std::unordered_set<std::string> output_args;
|
||||
std::unordered_set<std::string> external_output_args;
|
||||
|
||||
for (const auto& node_idx : cluster) {
|
||||
const auto& node = graph_viewer.GetNode(node_idx);
|
||||
|
||||
// Collect all inputs and outputs
|
||||
node->ForEachDef(
|
||||
[&input_args, &ordered_input_args, &output_args](const NodeArg& node_arg, bool is_input) {
|
||||
if (is_input) {
|
||||
if (!input_args.count(node_arg.Name())) {
|
||||
ordered_input_args.push_back(node_arg.Name());
|
||||
}
|
||||
input_args.insert(node_arg.Name());
|
||||
} else {
|
||||
output_args.insert(node_arg.Name());
|
||||
}
|
||||
},
|
||||
true);
|
||||
|
||||
// Check if output of this node is used by nodes outside this_cluster. If yes add this to cluster outputs
|
||||
for (auto it = node->OutputNodesBegin(); it != node->OutputNodesEnd(); ++it) {
|
||||
const auto& ext_node = graph_viewer.GetNode((*it).Index());
|
||||
|
||||
if (std::find(cluster.begin(), cluster.end(), ext_node->Index()) == cluster.end()) {
|
||||
// Node is external to this_cluster. Search through its inputs to find the output that is generated by this_cluster.
|
||||
std::set<std::string> ext_node_inputs;
|
||||
ext_node->ForEachDef(
|
||||
[&ext_node_inputs](const onnxruntime::NodeArg& arg, bool is_input) {
|
||||
if (is_input) {
|
||||
ext_node_inputs.insert(arg.Name());
|
||||
}
|
||||
},
|
||||
true);
|
||||
|
||||
for (const auto& out_def : node->OutputDefs()) {
|
||||
if (ext_node_inputs.find(out_def->Name()) != ext_node_inputs.end()) {
|
||||
external_output_args.insert(out_def->Name());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//Extract initializers used by this_cluster.
|
||||
std::unordered_set<std::string> original_graph_inputs;
|
||||
for (const auto& node_arg : graph_viewer.GetInputsIncludingInitializers()) {
|
||||
original_graph_inputs.insert(node_arg->Name());
|
||||
}
|
||||
|
||||
const auto& initializers = graph_viewer.GetAllInitializedTensors();
|
||||
std::vector<std::string> const_inputs;
|
||||
for (const auto& in_arg : ordered_input_args) {
|
||||
if ((initializers.count(in_arg) && !original_graph_inputs.count(in_arg)) ||
|
||||
ng_required_initializers.count(in_arg)) {
|
||||
const_inputs.push_back(in_arg);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto& in_arg : ordered_input_args) {
|
||||
if (!output_args.count(in_arg) &&
|
||||
!((initializers.count(in_arg) && !original_graph_inputs.count(in_arg)) ||
|
||||
ng_required_initializers.count(in_arg))) {
|
||||
cluster_inputs.push_back(in_arg);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto& in_arg : const_inputs) {
|
||||
cluster_inputs.push_back(in_arg);
|
||||
}
|
||||
|
||||
std::copy(external_output_args.begin(), external_output_args.end(), std::back_inserter(cluster_outputs));
|
||||
for (const auto& node_arg : graph_viewer.GetOutputs()) {
|
||||
const auto& name = node_arg->Name();
|
||||
if (output_args.count(name) && !external_output_args.count(name)) {
|
||||
cluster_outputs.push_back(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void AppendClusterToSubGraph(const std::vector<NodeIndex>& nodes,
|
||||
const std::vector<std::string>& inputs,
|
||||
const std::vector<std::string>& outputs,
|
||||
std::vector<std::unique_ptr<ComputeCapability>>& result) {
|
||||
static size_t op_counter = 0;
|
||||
|
||||
auto meta_def = onnxruntime::make_unique<IndexedSubGraph::MetaDef>();
|
||||
meta_def->name = "VitisAICustomOp_" + std::to_string(++op_counter);
|
||||
meta_def->domain = kVitisAIDomain;
|
||||
meta_def->since_version = 1;
|
||||
meta_def->status = ONNX_NAMESPACE::EXPERIMENTAL;
|
||||
meta_def->inputs = inputs;
|
||||
meta_def->outputs = outputs;
|
||||
|
||||
std::unique_ptr<IndexedSubGraph> sub_graph = onnxruntime::make_unique<IndexedSubGraph>();
|
||||
sub_graph->nodes = nodes;
|
||||
sub_graph->SetMetaDef(meta_def);
|
||||
result.push_back(onnxruntime::make_unique<ComputeCapability>(std::move(sub_graph)));
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::unique_ptr<ComputeCapability>>
|
||||
VitisAIExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
|
||||
const std::vector<const KernelRegistry*>& kernel_registries) const {
|
||||
ORT_UNUSED_PARAMETER(kernel_registries);
|
||||
|
||||
std::vector<std::unique_ptr<ComputeCapability>> result;
|
||||
|
||||
// Dump model Proto to file to pass it to pyxir
|
||||
auto logger = *GetLogger();
|
||||
|
||||
const Graph& node_graph = graph.GetGraph();
|
||||
const std::string& name_ = node_graph.Name();
|
||||
onnxruntime::Model model{name_, true, ModelMetaData{}, PathString{},
|
||||
IOnnxRuntimeOpSchemaRegistryList{},
|
||||
node_graph.DomainToVersionMap(),
|
||||
std::vector<ONNX_NAMESPACE::FunctionProto>(),
|
||||
logger};
|
||||
|
||||
ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
|
||||
model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
|
||||
|
||||
*(model_proto.mutable_graph()) = node_graph.ToGraphProto();
|
||||
|
||||
std::istringstream model_stream{model_proto.SerializeAsString()};
|
||||
|
||||
// Transform ONNX into Pyxir XGraph data structure
|
||||
XGraphHolder xg = pyxir::onnx::import_onnx_model(model_stream);
|
||||
|
||||
// Annotate the subgraph layers in the XGraph that can be executed on the
|
||||
// specified `backend_type_` target
|
||||
pyxir::partition(xg, std::vector<std::string>{backend_type_}, "");
|
||||
|
||||
// Next stuff
|
||||
if (graph.IsSubgraph()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Need access to model_path_
|
||||
for (const auto& tensor : graph.GetAllInitializedTensors()) {
|
||||
if (tensor.second->has_data_location() && tensor.second->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) {
|
||||
LOGS_DEFAULT(WARNING) << "VITIS-AI EP: Initializers with external data location are not currently supported";
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
std::unordered_set<std::string> required_initializers;
|
||||
const auto unsupported_nodes = GetUnsupportedNodeIndices(xg, backend_type_, graph, required_initializers);
|
||||
|
||||
const auto clusters = GetPartitionedClusters(graph.GetNodesInTopologicalOrder(), unsupported_nodes);
|
||||
|
||||
for (const auto& this_cluster : clusters) {
|
||||
std::vector<std::string> cluster_inputs, cluster_outputs;
|
||||
GetInputsOutputsOfCluster(graph, this_cluster, required_initializers, cluster_inputs, cluster_outputs);
|
||||
|
||||
if (!cluster_inputs.empty()) {
|
||||
AppendClusterToSubGraph(this_cluster, cluster_inputs, cluster_outputs, result);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
common::Status VitisAIExecutionProvider::Compile(const std::vector<onnxruntime::Node*>& fused_nodes,
|
||||
std::vector<NodeComputeInfo>& node_compute_funcs) {
|
||||
for (const auto& fused_node : fused_nodes)
|
||||
{
|
||||
NodeComputeInfo compute_info;
|
||||
compute_info.create_state_func = [this, fused_node, logger=GetLogger()](ComputeContext* context, FunctionState* state) {
|
||||
auto* p = new vitisai_ep::VitisAICustomOp(context, fused_node, backend_type_, logger);
|
||||
*state = p;
|
||||
return 0;
|
||||
};
|
||||
|
||||
compute_info.release_state_func = [](FunctionState state) {
|
||||
if (state)
|
||||
delete reinterpret_cast<onnxruntime::vitisai_ep::VitisAICustomOp*>(state);
|
||||
};
|
||||
|
||||
compute_info.compute_func = [](FunctionState state, const OrtApi* api, OrtKernelContext* context) {
|
||||
onnxruntime::vitisai_ep::VitisAICustomOp* custom_op = reinterpret_cast<onnxruntime::vitisai_ep::VitisAICustomOp*>(state);
|
||||
return custom_op->Compute(api, context);
|
||||
};
|
||||
|
||||
node_compute_funcs.push_back(compute_info);
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
// Copyright (c) Xilinx Inc. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ctime>
|
||||
#include "core/framework/execution_provider.h"
|
||||
#include "core/platform/ort_mutex.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
// Information needed to construct execution providers.
|
||||
struct VitisAIExecutionProviderInfo {
|
||||
int device_id{0};
|
||||
std::string backend_type;
|
||||
};
|
||||
|
||||
// Logical device representation.
|
||||
class VitisAIExecutionProvider : public IExecutionProvider {
|
||||
public:
|
||||
explicit VitisAIExecutionProvider(const VitisAIExecutionProviderInfo& info);
|
||||
~VitisAIExecutionProvider() = default;
|
||||
|
||||
std::vector<std::unique_ptr<ComputeCapability>>
|
||||
GetCapability(const onnxruntime::GraphViewer& graph,
|
||||
const std::vector<const KernelRegistry*>& /*kernel_registries*/) const override;
|
||||
|
||||
int GetDeviceId() const { return device_id_; }
|
||||
|
||||
common::Status Compile(const std::vector<onnxruntime::Node*>& fused_nodes,
|
||||
std::vector<NodeComputeInfo>& node_compute_funcs) override;
|
||||
|
||||
private:
|
||||
std::string backend_type_;
|
||||
int device_id_;
|
||||
};
|
||||
|
||||
} // namespace onnxruntime
|
||||
41
onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc
Executable file
41
onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc
Executable file
|
|
@ -0,0 +1,41 @@
|
|||
// Copyright (c) Xilinx Inc. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/providers/vitisai/vitisai_provider_factory.h"
|
||||
#include <atomic>
|
||||
#include "vitisai_execution_provider.h"
|
||||
#include "core/session/abi_session_options_impl.h"
|
||||
|
||||
using namespace onnxruntime;
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
struct VitisAIProviderFactory : IExecutionProviderFactory {
|
||||
VitisAIProviderFactory(std::string&& backend_type, int device_id)
|
||||
: backend_type_(std::move(backend_type)), device_id_(device_id) {}
|
||||
~VitisAIProviderFactory() = default;
|
||||
|
||||
std::unique_ptr<IExecutionProvider> CreateProvider() override;
|
||||
|
||||
private:
|
||||
const std::string backend_type_;
|
||||
int device_id_;
|
||||
};
|
||||
|
||||
std::unique_ptr<IExecutionProvider> VitisAIProviderFactory::CreateProvider() {
|
||||
VitisAIExecutionProviderInfo info;
|
||||
info.backend_type = backend_type_;
|
||||
info.device_id = device_id_;
|
||||
return onnxruntime::make_unique<VitisAIExecutionProvider>(info);
|
||||
}
|
||||
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_VITISAI(const char *backend_type, int device_id) {
|
||||
return std::make_shared<onnxruntime::VitisAIProviderFactory>(backend_type, device_id);
|
||||
}
|
||||
} // namespace onnxruntime
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_VITISAI, _In_ OrtSessionOptions* options, _In_ const char* backend_type, int device_id) {
|
||||
options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_VITISAI(backend_type, device_id));
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
|
|
@ -81,6 +81,13 @@
|
|||
#define BACKEND_NUPHAR ""
|
||||
#endif
|
||||
|
||||
#if USE_VITISAI
|
||||
#define BACKEND_VITISAI "-VITISAI"
|
||||
#include "core/providers/vitisai/vitisai_execution_provider.h"
|
||||
#else
|
||||
#define BACKEND_VITISAI ""
|
||||
#endif
|
||||
|
||||
#if USE_OPENBLAS
|
||||
#define BACKEND_OPENBLAS "-OPENBLAS"
|
||||
#else
|
||||
|
|
@ -113,6 +120,9 @@ std::string openvino_device;
|
|||
#include "core/providers/nuphar/nuphar_provider_factory.h"
|
||||
std::string nuphar_settings;
|
||||
#endif
|
||||
#ifdef USE_VITISAI
|
||||
#include "core/providers/vitisai/vitisai_provider_factory.h"
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CPU(int use_arena);
|
||||
|
|
@ -124,6 +134,8 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Dnnl(i
|
|||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_NGraph(const char* ng_backend_type);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVINO(const char* device);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nuphar(bool, const char*);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_VITISAI(const char *backend_type, int device_id);
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
|
|
@ -265,7 +277,7 @@ inline void RegisterExecutionProvider(InferenceSession* sess, onnxruntime::IExec
|
|||
const std::vector<std::string>& GetAllProviders() {
|
||||
static std::vector<std::string> all_providers = {kTensorrtExecutionProvider, kCudaExecutionProvider, kDnnlExecutionProvider,
|
||||
kNGraphExecutionProvider, kOpenVINOExecutionProvider, kNupharExecutionProvider,
|
||||
kCpuExecutionProvider};
|
||||
kVitisAIExecutionProvider, kCpuExecutionProvider};
|
||||
return all_providers;
|
||||
}
|
||||
|
||||
|
|
@ -289,6 +301,9 @@ const std::vector<std::string>& GetAvailableProviders() {
|
|||
#endif
|
||||
#ifdef USE_NUPHAR
|
||||
available_providers.push_back(kNupharExecutionProvider);
|
||||
#endif
|
||||
#ifdef USE_VITISAI
|
||||
available_providers.push_back(kVitisAIExecutionProvider);
|
||||
#endif
|
||||
return available_providers;
|
||||
};
|
||||
|
|
@ -328,6 +343,10 @@ void RegisterExecutionProviders(InferenceSession* sess, const std::vector<std::s
|
|||
#if USE_NUPHAR
|
||||
RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_Nuphar(true, nuphar_settings.c_str()));
|
||||
nuphar_settings.clear(); // clear nuphar_settings after use to avoid it being accidentally passed on to next session
|
||||
#endif
|
||||
} else if (type == kVitisAIExecutionProvider) {
|
||||
#if USE_VITISAI
|
||||
RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_VITISAI("dpuv1", 0));
|
||||
#endif
|
||||
} else {
|
||||
// unknown provider
|
||||
|
|
@ -416,6 +435,9 @@ void addGlobalMethods(py::module& m, const Environment& env) {
|
|||
#endif
|
||||
#ifdef USE_TENSORRT
|
||||
onnxruntime::CreateExecutionProviderFactory_Tensorrt(0)
|
||||
#endif
|
||||
#ifdef USE_VITISAI
|
||||
onnxruntime::CreateExecutionProviderFactory_VitisAI("DPU", 0),
|
||||
#endif
|
||||
};
|
||||
|
||||
|
|
|
|||
3
setup.py
3
setup.py
|
|
@ -65,6 +65,9 @@ elif '--use_dnnl' in sys.argv:
|
|||
elif '--use_nuphar' in sys.argv:
|
||||
package_name = 'onnxruntime-nuphar'
|
||||
sys.argv.remove('--use_nuphar')
|
||||
elif '--use_vitisai' in sys.argv:
|
||||
package_name = 'onnxruntime-vitisai'
|
||||
sys.argv.remove('--use_vitisai')
|
||||
# --use_acl is specified in build.py, but not parsed here
|
||||
|
||||
# PEP 513 defined manylinux1_x86_64 and manylinux1_i686
|
||||
|
|
|
|||
|
|
@ -274,6 +274,8 @@ def parse_arguments():
|
|||
"--enable_msinternal", action="store_true",
|
||||
help="Enable for Microsoft internal builds only.")
|
||||
parser.add_argument("--llvm_path", help="Path to llvm dir")
|
||||
parser.add_argument(
|
||||
"--use_vitisai", action='store_true', help="Build with Vitis-AI")
|
||||
parser.add_argument(
|
||||
"--use_nuphar", action='store_true', help="Build with nuphar")
|
||||
parser.add_argument(
|
||||
|
|
@ -573,6 +575,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home,
|
|||
"-Donnxruntime_USE_LLVM=" + ("ON" if args.use_llvm else "OFF"),
|
||||
"-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + (
|
||||
"ON" if args.enable_msinternal else "OFF"),
|
||||
"-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"),
|
||||
"-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"),
|
||||
"-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"),
|
||||
"-Donnxruntime_TENSORRT_HOME=" + (
|
||||
|
|
@ -629,7 +632,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home,
|
|||
# nGraph, TensorRT and OpenVINO providers currently only supports
|
||||
# full_protobuf option.
|
||||
if (args.use_full_protobuf or args.use_ngraph or args.use_tensorrt or
|
||||
args.use_openvino or args.gen_doc):
|
||||
args.use_openvino or args.use_vitisai or args.gen_doc):
|
||||
cmake_args += [
|
||||
"-Donnxruntime_USE_FULL_PROTOBUF=ON",
|
||||
"-DProtobuf_USE_STATIC_LIBS=ON"
|
||||
|
|
@ -1351,9 +1354,8 @@ def nuphar_run_python_tests(build_dir, configs):
|
|||
|
||||
def build_python_wheel(
|
||||
source_dir, build_dir, configs, use_cuda, use_ngraph, use_dnnl,
|
||||
use_tensorrt, use_openvino, use_nuphar, wheel_name_suffix, use_acl,
|
||||
nightly_build=False,
|
||||
featurizers_build=False):
|
||||
use_tensorrt, use_openvino, use_nuphar, use_vitisai, wheel_name_suffix,
|
||||
use_acl, nightly_build=False, featurizers_build=False):
|
||||
for config in configs:
|
||||
cwd = get_config_build_dir(build_dir, config)
|
||||
if is_windows():
|
||||
|
|
@ -1393,6 +1395,8 @@ def build_python_wheel(
|
|||
args.append('--use_dnnl')
|
||||
elif use_nuphar:
|
||||
args.append('--use_nuphar')
|
||||
elif use_vitisai:
|
||||
args.append('--use_vitisai')
|
||||
elif use_acl:
|
||||
args.append('--use_acl')
|
||||
|
||||
|
|
@ -1737,6 +1741,7 @@ def main():
|
|||
args.use_tensorrt,
|
||||
args.use_openvino,
|
||||
args.use_nuphar,
|
||||
args.use_vitisai,
|
||||
args.wheel_name_suffix,
|
||||
args.use_acl,
|
||||
nightly_build=nightly_build,
|
||||
|
|
|
|||
Loading…
Reference in a new issue