diff --git a/BUILD.md b/BUILD.md index ffc08f1ca5..fd075c7833 100644 --- a/BUILD.md +++ b/BUILD.md @@ -20,6 +20,7 @@ * [DirectML](#DirectML) * [ARM Compute Library](#ARM-Compute-Library) * [Rockchip RKNPU](#RKNPU) + * [Xilinx Vitis-AI](#Vitis-AI) * Options * [OpenMP](#OpenMP) * [OpenBLAS](#OpenBLAS) @@ -517,8 +518,22 @@ See more information on the RKNPU Execution Provider [here](./docs/execution_pro onnxruntime_test_all rknpu_ddk/lib64/librknpu_ddk.so ``` + --- +### Vitis-AI +See more information on the Xilinx Vitis-AI execution provider [here](./docs/execution_providers/Vitis-AI-ExecutionProvider.md). + +For instructions to setup the hardware environment: [Hardware setup](./docs/execution_providers/Vitis-AI-ExecutionProvider.md#Hardware-setup) + +#### Linux + +``` +./build.sh --use_vitisai +``` +#### Notes +The Vitis-AI execution provider is only supported on Linux. + ## Options ### OpenMP #### Build Instructions diff --git a/README.md b/README.md index af676bd523..626f389b70 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ For production scenarios, it's strongly recommended to build only from an [offic |CPU|GPU|IoT/Edge/Mobile|Other| |---|---|---|---| -||||| +||||| * [Roadmap: Upcoming accelerators](./docs/Roadmap.md#accelerators-and-execution-providers) * [Extensibility: Add an execution provider](docs/AddingExecutionProvider.md) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 437214f507..f515264688 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -72,6 +72,7 @@ option(onnxruntime_USE_OPENMP "Build with OpenMP support" OFF) option(onnxruntime_BUILD_SHARED_LIB "Build a shared library" OFF) option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable microsoft internal only code" OFF) option(onnxruntime_USE_NUPHAR "Build with Nuphar" OFF) +option(onnxruntime_USE_VITISAI "Build with Vitis-AI" OFF) option(onnxruntime_USE_TENSORRT "Build with TensorRT support" OFF) option(onnxruntime_ENABLE_LTO "Enable link time optimization" OFF) option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF) @@ -762,6 +763,17 @@ if(onnxruntime_USE_OPENVINO) endif() +if (onnxruntime_USE_VITISAI) + if(WIN32) + message(FATAL_ERROR "Vitis-AI execution provider is not supported on Windows.") + else() + add_definitions(-DUSE_VITISAI=1) + include(pyxir) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES pyxir) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES pyxir) + endif() +endif() + if (onnxruntime_USE_OPENBLAS) add_definitions(-DUSE_OPENBLAS=1) if (WIN32) diff --git a/cmake/external/pyxir.cmake b/cmake/external/pyxir.cmake new file mode 100644 index 0000000000..03f462366f --- /dev/null +++ b/cmake/external/pyxir.cmake @@ -0,0 +1,28 @@ +# Copyright(C) Xilinx Inc. +# Licensed under the MIT License + +set(PYXIR_SHARED_LIB libpyxir.so) + +if(PYTHONINTERP_FOUND) + execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" + "import pyxir as px; print(px.get_include_dir()); print(px.get_lib_dir());" + RESULT_VARIABLE __result + OUTPUT_VARIABLE __output + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(__result MATCHES 0) + string(REGEX REPLACE ";" "\\\\;" __values ${__output}) + string(REGEX REPLACE "\r?\n" ";" __values ${__values}) + list(GET __values 0 PYXIR_INCLUDE_DIR) + list(GET __values 1 PYXIR_LIB_DIR) + endif() +else() + message(STATUS "To find Pyxir, Python interpretater is required to be found.") +endif() + +add_library(pyxir SHARED IMPORTED) +message("-- Found Pyxir lib: ${PYXIR_LIB_DIR}/${PYXIR_SHARED_LIB}") +set_property(TARGET pyxir PROPERTY IMPORTED_LOCATION ${PYXIR_LIB_DIR}/${PYXIR_SHARED_LIB}) + +message("-- Found Pyxir include: ${PYXIR_INCLUDE_DIR}") +include_directories(${PYXIR_INCLUDE_DIR}) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index d2ca120f68..6815227589 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -84,6 +84,7 @@ target_link_libraries(onnxruntime PRIVATE ${PROVIDERS_TENSORRT} ${PROVIDERS_OPENVINO} ${PROVIDERS_NUPHAR} + ${PROVIDERS_VITISAI} ${PROVIDERS_DML} ${PROVIDERS_ACL} ${onnxruntime_winml} diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index e9f5eb6bd1..9998321a45 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -43,6 +43,10 @@ if(onnxruntime_USE_NUPHAR) set(PROVIDERS_NUPHAR onnxruntime_providers_nuphar) list(APPEND ONNXRUNTIME_PROVIDER_NAMES nuphar) endif() +if(onnxruntime_USE_VITISAI) + set(PROVIDERS_VITISAI onnxruntime_providers_vitisai) + list(APPEND ONNXRUNTIME_PROVIDER_NAMES vitisai) +endif() if(onnxruntime_USE_CUDA) set(PROVIDERS_CUDA onnxruntime_providers_cuda) list(APPEND ONNXRUNTIME_PROVIDER_NAMES cuda) @@ -413,6 +417,22 @@ if (onnxruntime_USE_NUPHAR) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/nuphar DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) endif() +if (onnxruntime_USE_VITISAI) + file(GLOB_RECURSE onnxruntime_providers_vitisai_cc_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/core/providers/vitisai/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/vitisai/*.cc" + ) + + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vitisai_cc_srcs}) + add_library(onnxruntime_providers_vitisai ${onnxruntime_providers_vitisai_cc_srcs}) + onnxruntime_add_include_to_target(onnxruntime_providers_vitisai onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf) + add_dependencies(onnxruntime_providers_vitisai ${onnxruntime_EXTERNAL_DEPENDENCIES}) + set_target_properties(onnxruntime_providers_vitisai PROPERTIES FOLDER "ONNXRuntime") + target_include_directories(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${VITISAI_INCLUDE_DIR}) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/vitisai DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) + set_target_properties(onnxruntime_providers_vitisai PROPERTIES LINKER_LANGUAGE CXX) +endif() + if (onnxruntime_USE_OPENVINO) include_directories("${CMAKE_CURRENT_BINARY_DIR}/onnx") diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index ef87de09f8..5edaa75bc7 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -88,6 +88,7 @@ set(onnxruntime_pybind11_state_libs ${PROVIDERS_NGRAPH} ${PROVIDERS_OPENVINO} ${PROVIDERS_NUPHAR} + ${PROVIDERS_VITISAI} ${PROVIDERS_NNAPI} ${PROVIDERS_RKNPU} ${PROVIDERS_DML} diff --git a/dockerfiles/Dockerfile.vitisai b/dockerfiles/Dockerfile.vitisai new file mode 100644 index 0000000000..c8b253e78f --- /dev/null +++ b/dockerfiles/Dockerfile.vitisai @@ -0,0 +1,37 @@ +# -------------------------------------------------------------- +# Copyright(C) Xilinx Inc. +# Licensed under the MIT License. +# -------------------------------------------------------------- +# Dockerfile to run ONNXRuntime with Vitis-AI integration + +FROM xilinx/vitis-ai:latest + +ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime +ARG ONNXRUNTIME_BRANCH=master + +ARG PYXIR_REPO=https://github.com/Xilinx/pyxir +ARG PYXIR_BRANCH=master + +RUN apt-get update &&\ + apt-get install -y sudo git bash + +ENV PATH /code/cmake-3.14.3-Linux-x86_64/bin:$PATH +ENV LD_LIBRARY_PATH /opt/xilinx/xrt/lib:$LD_LIBRARY_PATH + +WORKDIR /code +RUN . $VAI_ROOT/conda/etc/profile.d/conda.sh &&\ + conda activate vitis-ai-tensorflow &&\ + git clone --single-branch --branch ${PYXIR_BRANCH} --recursive ${PYXIR_REPO} pyxir &&\ + cd pyxir && python3 setup.py install --use_vai_rt +RUN . $VAI_ROOT/conda/etc/profile.d/conda.sh &&\ + conda activate vitis-ai-tensorflow &&\ + git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime &&\ + /bin/sh onnxruntime/dockerfiles/scripts/install_common_deps.sh &&\ + cp onnxruntime/docs/Privacy.md /code/Privacy.md &&\ + cp onnxruntime/dockerfiles/LICENSE-IMAGE.txt /code/LICENSE-IMAGE.txt &&\ + cp onnxruntime/ThirdPartyNotices.txt /code/ThirdPartyNotices.txt &&\ + cd onnxruntime &&\ + /bin/sh ./build.sh --config RelWithDebInfo --enable_pybind --build_wheel --use_vitisai --parallel --update --build --build_shared_lib &&\ + pip install /code/onnxruntime/build/Linux/RelWithDebInfo/dist/*-linux_x86_64.whl &&\ + cd .. &&\ + rm -rf onnxruntime cmake-3.14.3-Linux-x86_64 diff --git a/dockerfiles/scripts/docker_run_vitisai.sh b/dockerfiles/scripts/docker_run_vitisai.sh new file mode 100755 index 0000000000..43fc08a7df --- /dev/null +++ b/dockerfiles/scripts/docker_run_vitisai.sh @@ -0,0 +1,33 @@ +# -------------------------------------------------------------- +# Copyright(C) Xilinx Inc. +# Licensed under the MIT License. +# -------------------------------------------------------------- + +user=`whoami` +uid=`id -u` +gid=`id -g` + +xclmgmt_driver="$(find /dev -name xclmgmt\*)" +docker_devices="" +for i in ${xclmgmt_driver} ; +do + docker_devices+="--device=$i " +done + +render_driver="$(find /dev/dri -name renderD\*)" +for i in ${render_driver} ; +do + docker_devices+="--device=$i " +done + +docker run \ + $docker_devices \ + -v /opt/xilinx/dsa:/opt/xilinx/dsa \ + -v /opt/xilinx/overlaybins:/opt/xilinx/overlaybins \ + -e USER=$user -e UID=$uid -e GID=$gid \ + -v $PWD:/workspace \ + -w /workspace \ + -it \ + --rm \ + --network=host \ + onnxruntime-vitisai diff --git a/docs/execution_providers/Vitis-AI-ExecutionProvider.md b/docs/execution_providers/Vitis-AI-ExecutionProvider.md new file mode 100644 index 0000000000..289010b403 --- /dev/null +++ b/docs/execution_providers/Vitis-AI-ExecutionProvider.md @@ -0,0 +1,100 @@ +

+ +

+ +# Vitis-AI Execution Provider + +[Vitis-AI](https://github.com/Xilinx/Vitis-AI) is Xilinx's development stack for hardware-accelerated AI inference on Xilinx platforms, including both edge devices and Alveo cards. It consists of optimized IP, tools, libraries, models, and example designs. It is designed with high efficiency and ease of use in mind, unleashing the full potential of AI acceleration on Xilinx FPGA and ACAP. + +## Build + +For build instructions, please see the [BUILD page](../../BUILD.md#Vitis-AI). Please setup the hardware environment before starting the build: [Hardware setup](#Hardware-setup). + +### System requirements + +The following table lists system requirements for running docker containers as well as Alveo cards. + + +| **Component** | **Requirement** | +|-----------------------------------------------------|------------------------------------------------------------| +| Motherboard | PCI Express 3\.0\-compliant with one dual\-width x16 slot | +| System Power Supply | 225W | +| Operating System | Ubuntu 16\.04, 18\.04 | +| | CentOS 7\.4, 7\.5 | +| | RHEL 7\.4, 7\.5 | +| CPU | Intel i3/i5/i7/i9/Xeon 64-bit CPU | +| GPU \(Optional to accelerate quantization\) | NVIDIA GPU with a compute capability > 3.0 | +| CUDA Driver \(Optional to accelerate quantization\) | nvidia\-410 | +| FPGA | Xilinx Alveo U200 or U250 | +| Docker Version | 19\.03\.1 | + +### Hardware setup + +1. Clone the Vitis AI repository: + ``` + git clone https://github.com/xilinx/vitis-ai + ``` +2. Install the Docker, and add the user to the docker group. Link the user to docker installation instructions from the following docker's website: + * https://docs.docker.com/install/linux/docker-ce/ubuntu/ + * https://docs.docker.com/install/linux/docker-ce/centos/ + * https://docs.docker.com/install/linux/linux-postinstall/ +3. Any GPU instructions will have to be separated from Vitis AI. +4. Set up Vitis AI to target Alveo cards. To target Alveo cards with Vitis AI for machine learning workloads, you must install the following software components: + * Xilinx Runtime (XRT) + * Alveo Deployment Shells (DSAs) + * Xilinx Resource Manager (XRM) (xbutler) + * Xilinx Overlaybins (Accelerators to Dynamically Load - binary programming files) + + While it is possible to install all of these software components individually, a script has been provided to automatically install them at once. To do so: + * Run the following commands: + ``` + cd Vitis-AI/alveo/packages + sudo su + ./install.sh + ``` + * Power cycle the system. +5. Build and start the ONNXRuntime Vitis-AI Docker Container. + ``` + cd {onnxruntime-root}/dockerfiles + docker build -t onnxruntime-vitisai -f Dockerfile.vitisai . + ./scripts/docker_run_vitisai.sh + ``` + + Setup inside container + ``` + source /opt/xilinx/xrt/setup.sh + conda activate vitis-ai-tensorflow + ``` + + + +### Samples + +For python, you can base yourself on the following example: + +``` +# Import pyxir before onnxruntime +import pyxir +import pyxir.frontend.onnx +import pyxir.contrib.dpuv1.dpuv1 + +import onnxruntime + +# Add other imports +# ... + +# Load inputs and do preprocessing +# ... + +# Create an inference session using the Vitis-AI execution provider +session = onnxruntime.InferenceSession('[model_file].onnx', None,["VitisAIExecutionProvider"]) + +# First N (default = 128) inputs are used for quantization calibration and will +# be executed on the CPU +imput_name = [...] +outputs = [session.run([], {input_name: calib_inputs[i]})[0] for i in range(128)] + +# Afterwards, computations will be accelerated on the FPGA +input_data = [...] +result = session.run([], {input_name: input_data}) +``` diff --git a/docs/execution_providers/images/Vitis-AI.png b/docs/execution_providers/images/Vitis-AI.png new file mode 100644 index 0000000000..88ffbf5e20 Binary files /dev/null and b/docs/execution_providers/images/Vitis-AI.png differ diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h index ca4d7ee5ed..b491b5f124 100644 --- a/include/onnxruntime/core/graph/constants.h +++ b/include/onnxruntime/core/graph/constants.h @@ -22,12 +22,14 @@ constexpr const char* kMSNchwcDomain = "com.microsoft.nchwc"; constexpr const char* kMSFeaturizersDomain = "com.microsoft.mlfeaturizers"; constexpr const char* kMSDmlDomain = "com.microsoft.dml"; constexpr const char* kNGraphDomain = "com.intel.ai"; +constexpr const char* kVitisAIDomain = "com.xilinx"; constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider"; constexpr const char* kCudaExecutionProvider = "CUDAExecutionProvider"; constexpr const char* kDnnlExecutionProvider = "DnnlExecutionProvider"; constexpr const char* kNGraphExecutionProvider = "NGRAPHExecutionProvider"; constexpr const char* kOpenVINOExecutionProvider = "OpenVINOExecutionProvider"; constexpr const char* kNupharExecutionProvider = "NupharExecutionProvider"; +constexpr const char* kVitisAIExecutionProvider = "VitisAIExecutionProvider"; constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider"; constexpr const char* kNnapiExecutionProvider = "NnapiExecutionProvider"; constexpr const char* kRknpuExecutionProvider = "RknpuExecutionProvider"; diff --git a/include/onnxruntime/core/providers/vitisai/vitisai_provider_factory.h b/include/onnxruntime/core/providers/vitisai/vitisai_provider_factory.h new file mode 100644 index 0000000000..eb4765bddc --- /dev/null +++ b/include/onnxruntime/core/providers/vitisai/vitisai_provider_factory.h @@ -0,0 +1,17 @@ +// Copyright (c) Xilinx Inc. +// Licensed under the MIT License. + +#include "onnxruntime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \param use_arena zero: false. non-zero: true. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_VITISAI, _In_ OrtSessionOptions* options, const char *backend_type, int device_id); + +#ifdef __cplusplus +} +#endif diff --git a/onnxruntime/core/framework/utils.cc b/onnxruntime/core/framework/utils.cc index ac064e9e6d..66b33c84f8 100644 --- a/onnxruntime/core/framework/utils.cc +++ b/onnxruntime/core/framework/utils.cc @@ -99,6 +99,7 @@ bool ProviderIsCpuBased(const std::string& provider_type) { provider_type == onnxruntime::kDnnlExecutionProvider || provider_type == onnxruntime::kNGraphExecutionProvider || provider_type == onnxruntime::kNupharExecutionProvider || + provider_type == onnxruntime::kVitisAIExecutionProvider || provider_type == onnxruntime::kOpenVINOExecutionProvider || provider_type == onnxruntime::kNnapiExecutionProvider || provider_type == onnxruntime::kRknpuExecutionProvider; diff --git a/onnxruntime/core/optimizer/transformer_memcpy.cc b/onnxruntime/core/optimizer/transformer_memcpy.cc index 5767c20c96..dbd40950d9 100644 --- a/onnxruntime/core/optimizer/transformer_memcpy.cc +++ b/onnxruntime/core/optimizer/transformer_memcpy.cc @@ -73,6 +73,7 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr provider != onnxruntime::kDnnlExecutionProvider && provider != onnxruntime::kNGraphExecutionProvider && provider != onnxruntime::kNupharExecutionProvider && + provider != onnxruntime::kVitisAIExecutionProvider && provider != onnxruntime::kOpenVINOExecutionProvider && provider != onnxruntime::kAclExecutionProvider) { TransformerMemcpyImpl copy_impl(graph, provider); @@ -219,6 +220,7 @@ void TransformerMemcpyImpl::ProcessDefs(onnxruntime::Node& node, const KernelReg } else if (node_provider_type != kCudaExecutionProvider && node_provider_type != kTensorrtExecutionProvider) { // TODO: copy between devices? i.e. multiple GPUs if (node_provider_type != onnxruntime::kCpuExecutionProvider && + node_provider_type != onnxruntime::kVitisAIExecutionProvider && node_provider_type != onnxruntime::kNGraphExecutionProvider && !node_provider_type.empty()) { ORT_THROW("Execution type '", node_provider_type, "' doesn't support memcpy "); } diff --git a/onnxruntime/core/providers/vitisai/symbols.txt b/onnxruntime/core/providers/vitisai/symbols.txt new file mode 100755 index 0000000000..4b29ec2d43 --- /dev/null +++ b/onnxruntime/core/providers/vitisai/symbols.txt @@ -0,0 +1 @@ +OrtSessionOptionsAppendExecutionProvider_VITISAI diff --git a/onnxruntime/core/providers/vitisai/vitisai_custom_op.cc b/onnxruntime/core/providers/vitisai/vitisai_custom_op.cc new file mode 100644 index 0000000000..4cc163ced4 --- /dev/null +++ b/onnxruntime/core/providers/vitisai/vitisai_custom_op.cc @@ -0,0 +1,170 @@ +// Copyright(C) Xilinx Inc. +// Licensed under the MIT License + +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable : 4244 4245) +#elif __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#if defined(_MSC_VER) +#pragma warning(default : 4244 4245) +#elif __GNUC__ +#pragma GCC diagnostic pop +#endif + +#include + +#include +#include +#include + +#include "vitisai_custom_op.h" +#include "core/graph/graph_viewer.h" +#include "core/graph/model.h" +#include "core/common/logging/logging.h" +#include "core/session/onnxruntime_cxx_api.h" + +namespace onnxruntime { +namespace vitisai_ep { + +static ONNX_NAMESPACE::ModelProto GetModelProtoFromFusedNode(const onnxruntime::Node* fused_node, + const logging::Logger& logger) { + const auto* node_function = fused_node->GetFunctionBody(); + + ORT_ENFORCE(node_function != nullptr, "Could not extract function body for node: ", + fused_node->Name()); + + const Graph& node_subgraph = node_function->Body(); + onnxruntime::Model model{node_subgraph.Name(), true, ModelMetaData{}, PathString{}, + IOnnxRuntimeOpSchemaRegistryList{}, node_subgraph.DomainToVersionMap(), + std::vector(), logger}; + + ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); + model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + + *(model_proto.mutable_graph()) = node_subgraph.ToGraphProto(); + + return model_proto; +} + +VitisAICustomOp::VitisAICustomOp(const ComputeContext* context, + const onnxruntime::Node* fused_node, + const std::string &backend_type, + const logging::Logger* logger) + : backend_type_(backend_type) +{ + SetLogger(logger); + + allocate_func_ = context->allocate_func; + release_func_ = context->release_func; + allocator_ = context->allocator_handle; + name_ = context->node_name; + + model_proto_ = GetModelProtoFromFusedNode(fused_node, *GetLogger()); + + std::istringstream model_stream{model_proto_.SerializeAsString()}; + + xg_ = pyxir::onnx::import_onnx_model(model_stream); + pyxir::partition(xg_, std::vector{backend_type_}, ""); + + auto input_defs = fused_node->InputDefs(); + for (auto idef : input_defs) { + // std::cout << "DPU input def: " << idef->Name() << std::endl; + in_tensor_names_.push_back(idef->Name()); + } + + auto output_defs = fused_node->OutputDefs(); + for (auto odef : output_defs) { + // std::cout << "DPU output def: " << odef->Name() << std::endl; + out_tensor_names_.push_back(odef->Name()); + } + + pyxir::RunOptionsHolder run_options(new pyxir::runtime::RunOptions()); + run_options->online_quantization = true; + rt_mod_ = pyxir::build_rt(xg_, backend_type_, in_tensor_names_, out_tensor_names_, + "vai", run_options); +} + +VitisAICustomOp::~VitisAICustomOp() {} + + +Status VitisAICustomOp::Compute(const OrtApi* api, OrtKernelContext* context) const { + Ort::CustomOpApi ort{*api}; + const unsigned num_inputs = (unsigned) xg_->get_nb_inputs(); + + ssize_t batch_size = 1; + std::vector in_tensors; + std::vector out_tensors; + + // Initialize input tensors. + try { + for (unsigned i = 0; i < num_inputs; ++i) { + // std::cout << "Input name: " << in_tensor_names_[i]; + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, i); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_type = ort.GetTensorElementType(tensor_info); + auto ort_shape = ort.GetTensorShape(tensor_info); + std::vector tensor_shape{ort_shape.begin(), ort_shape.end()}; + batch_size = tensor_shape[0]; + + if (tensor_type != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, + "VITIS-AI EP input onnx tensor data type: " + std::to_string(tensor_type) + " not supported."); + + void* input_data = const_cast(ort.GetTensorData(input_tensor)); + in_tensors.push_back(std::shared_ptr( + new pyxir::XBuffer(input_data, 4, "f", tensor_shape.size(), tensor_shape, + false, false))); + } + } catch (const std::exception& exp) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Exception while copying input data to Pyxir: " + std::string(exp.what())); + } catch (...) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Unknown exception while copying input data to Pyxir"); + } + + // Initialize output tensors + try { + for (unsigned i = 0; i < out_tensor_names_.size(); ++i) { + auto shape = xg_->get(out_tensor_names_[i])->shapes[0]; + std::vector out_shape{shape.begin(), shape.end()}; + out_shape[0] = batch_size; + std::vector ort_shape{out_shape.begin(), out_shape.end()}; + + OrtValue* output_tensor = ort.KernelContext_GetOutput(context, i, ort_shape.data(), ort_shape.size()); + auto tensor_info = ort.GetTensorTypeAndShape(output_tensor); + auto tensor_type = ort.GetTensorElementType(tensor_info); + if (tensor_type != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, + "VITIS-AI EP input onnx tensor data type: " + std::to_string(tensor_type) + " not supported."); + + void* output_data = ort.GetTensorMutableData(output_tensor); + out_tensors.push_back(std::shared_ptr( + new pyxir::XBuffer(output_data, 4, "f", out_shape.size(), out_shape, + false, false))); + } + } catch (const std::exception& exp) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Exception while creating Pyxir output Tensor: " + std::string(exp.what())); + } catch (...) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Unknown exception while creating Pyxir output Tensor"); + } + + // Run the graph through Vitis-AI Pyxir + try { + // std::lock_guard lock(compute_lock_); + rt_mod_->execute(in_tensors, out_tensors); + } catch (const std::exception& exp) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Exception while executing Pyxir computation: " + std::string(exp.what())); + } catch (...) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, name_ + ": Unknown exception while executing Pyxir computation"); + } + + return Status::OK(); +} + +} // namespace vitisai_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vitisai/vitisai_custom_op.h b/onnxruntime/core/providers/vitisai/vitisai_custom_op.h new file mode 100644 index 0000000000..26dd507160 --- /dev/null +++ b/onnxruntime/core/providers/vitisai/vitisai_custom_op.h @@ -0,0 +1,74 @@ +// Copyright(C) Xilinx Inc. +// Licensed under the MIT License + +#pragma once + +#if defined(_MSC_VER) +#pragma warning(disable : 4244 4245) +#elif __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#if defined(_MSC_VER) +#pragma warning(default : 4244 4245) +#elif __GNUC__ +#pragma GCC diagnostic pop +#endif + +#include + +#include "core/session/onnxruntime_c_api.h" +#include "core/framework/func_api.h" +#include "core/graph/graph_viewer.h" +#include "core/common/logging/logging.h" +#include "core/graph/onnx_protobuf.h" + +namespace onnxruntime { +namespace vitisai_ep { + +class VitisAICustomOp { + public: + VitisAICustomOp(const ComputeContext* context, + const onnxruntime::Node* fused_node, + const std::string &backend_type, + const logging::Logger* logger); + + Status Compute(const OrtApi* api, OrtKernelContext* context) const; + + ~VitisAICustomOp(); + + void SetLogger(const logging::Logger* logger) { + logger_ = logger; + } + + const logging::Logger* GetLogger() const { + return logger_; + } + + private: + Status Initialize(const OrtApi* api, OrtKernelContext* context) const; + + std::vector in_tensor_names_; + std::vector out_tensor_names_; + pyxir::XGraphHolder xg_; + + std::string backend_type_; + + pyxir::RtModHolder rt_mod_ = nullptr; + + AllocateFunc allocate_func_ = nullptr; + + DestroyFunc release_func_ = nullptr; + + AllocatorHandle allocator_ = nullptr; + + std::string name_; + + mutable std::mutex compute_lock_; + + const logging::Logger* logger_ = nullptr; + + ONNX_NAMESPACE::ModelProto model_proto_; +}; +} // namespace vitisai_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc new file mode 100644 index 0000000000..74d6934cc5 --- /dev/null +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc @@ -0,0 +1,324 @@ +// Copyright (c) Xilinx Inc. All rights reserved. +// Licensed under the MIT License. + +#include +#include + +#include +#include + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/framework/compute_capability.h" +#include "core/framework/allocatormgr.h" +#include "core/framework/kernel_registry.h" +#include "core/graph/graph_viewer.h" +#include "core/graph/model.h" +#include "vitisai_execution_provider.h" +#include "vitisai_custom_op.h" + +#define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, std::min(destsz, srcsz)) + +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::logging; + +namespace onnxruntime { + +constexpr const char* PREFIX = "VITISAI"; + +typedef std::shared_ptr XGraphHolder; +typedef std::shared_ptr XLayerHolder; + +VitisAIExecutionProvider::VitisAIExecutionProvider(const VitisAIExecutionProviderInfo& info) + : IExecutionProvider{onnxruntime::kVitisAIExecutionProvider}, backend_type_(info.backend_type), + device_id_(info.device_id) { + + auto default_allocator_factory = [](int) { + auto memory_info = onnxruntime::make_unique(PREFIX, OrtAllocatorType::OrtDeviceAllocator); + return onnxruntime::make_unique(std::move(memory_info)); + }; + + DeviceAllocatorRegistrationInfo default_memory_info{ + OrtMemTypeDefault, + std::move(default_allocator_factory), + std::numeric_limits::max() + }; + + InsertAllocator(CreateAllocator(default_memory_info)); +} + + +static std::vector +GetUnsupportedNodeIndices(const XGraphHolder &xg, const std::string &backend_type, + const GraphViewer& graph_viewer, + /*out*/ std::unordered_set& required_initializers) { + + // Retrieve + std::set supported_tensors; + for (auto &xl_name : xg->get_layer_names()) { + XLayerHolder xl = xg->get(xl_name); + if (xl->target == backend_type) + supported_tensors.insert(xl->get_attr("onnx_id").get_string()); + } + + std::vector unsupported_nodes_idx; + + for (const auto& node_idx : graph_viewer.GetNodesInTopologicalOrder()) { + ConstPointerContainer> node_args + = graph_viewer.GetNode(node_idx)->OutputDefs(); + + bool is_node_supported = false; + for (ConstPointerContainer>::ConstIterator it = + node_args.begin(); it != node_args.end(); ++it) { + if (supported_tensors.find((*it)->Name()) != supported_tensors.end()) { + is_node_supported = true; + } else if (is_node_supported) { + // Some output tensors are supported but not others, + // should not happen + LOGS_DEFAULT(FATAL) << "VITIS-AI EP: Found node output tensor " + << (*it)->Name() << " which is partially supported by " + << " DPU accelerator. This is an invalid case"; + } + } + + if (is_node_supported) { + // Collect inputs that are initializers + graph_viewer.GetNode(node_idx)->ForEachDef([&required_initializers, &graph_viewer](const onnxruntime::NodeArg& node_arg, bool is_input) { + if(is_input && graph_viewer.GetAllInitializedTensors().count(node_arg.Name())) { + required_initializers.insert(node_arg.Name()); + } }, true); + } else { + unsupported_nodes_idx.push_back(node_idx); + } + } + + return unsupported_nodes_idx; +} + +/** + * Returns a vector clusters(or node_idx). For each unsupported node, the graph is split into 3 parts. + * supported_cluster + (UNsupported_node + rest_of_the_graph). This functions returns vector of all supported_clusters by DPU + */ +static std::vector> +GetPartitionedClusters(const std::vector& topological_order, const std::vector& unsupported_nodes) { + std::vector> clusters; + + auto prev = topological_order.begin(); + + for (const auto& unsup_node : unsupported_nodes) { + auto it = std::find(prev, topological_order.end(), unsup_node); + // Create a cluster vector[supported_node_idx, unsupported_node_idx) and append it to return list. + std::vector this_cluster{prev, it}; + if (!this_cluster.empty()) { + clusters.push_back(std::move(this_cluster)); + } + // Point prev to node idx past this unsuported node. + prev = ++it; + } + + //Tail + std::vector this_cluster{prev, topological_order.end()}; + if (!this_cluster.empty()) { + clusters.push_back(std::move(this_cluster)); + } + + return clusters; +} + +static void GetInputsOutputsOfCluster(const GraphViewer& graph_viewer, + const std::vector& cluster, + const std::unordered_set& ng_required_initializers, + /*out*/ std::vector& cluster_inputs, + /*out*/ std::vector& cluster_outputs) { + std::unordered_set input_args; + std::vector ordered_input_args; + std::unordered_set output_args; + std::unordered_set external_output_args; + + for (const auto& node_idx : cluster) { + const auto& node = graph_viewer.GetNode(node_idx); + + // Collect all inputs and outputs + node->ForEachDef( + [&input_args, &ordered_input_args, &output_args](const NodeArg& node_arg, bool is_input) { + if (is_input) { + if (!input_args.count(node_arg.Name())) { + ordered_input_args.push_back(node_arg.Name()); + } + input_args.insert(node_arg.Name()); + } else { + output_args.insert(node_arg.Name()); + } + }, + true); + + // Check if output of this node is used by nodes outside this_cluster. If yes add this to cluster outputs + for (auto it = node->OutputNodesBegin(); it != node->OutputNodesEnd(); ++it) { + const auto& ext_node = graph_viewer.GetNode((*it).Index()); + + if (std::find(cluster.begin(), cluster.end(), ext_node->Index()) == cluster.end()) { + // Node is external to this_cluster. Search through its inputs to find the output that is generated by this_cluster. + std::set ext_node_inputs; + ext_node->ForEachDef( + [&ext_node_inputs](const onnxruntime::NodeArg& arg, bool is_input) { + if (is_input) { + ext_node_inputs.insert(arg.Name()); + } + }, + true); + + for (const auto& out_def : node->OutputDefs()) { + if (ext_node_inputs.find(out_def->Name()) != ext_node_inputs.end()) { + external_output_args.insert(out_def->Name()); + } + } + } + } + } + + //Extract initializers used by this_cluster. + std::unordered_set original_graph_inputs; + for (const auto& node_arg : graph_viewer.GetInputsIncludingInitializers()) { + original_graph_inputs.insert(node_arg->Name()); + } + + const auto& initializers = graph_viewer.GetAllInitializedTensors(); + std::vector const_inputs; + for (const auto& in_arg : ordered_input_args) { + if ((initializers.count(in_arg) && !original_graph_inputs.count(in_arg)) || + ng_required_initializers.count(in_arg)) { + const_inputs.push_back(in_arg); + } + } + + for (const auto& in_arg : ordered_input_args) { + if (!output_args.count(in_arg) && + !((initializers.count(in_arg) && !original_graph_inputs.count(in_arg)) || + ng_required_initializers.count(in_arg))) { + cluster_inputs.push_back(in_arg); + } + } + + for (const auto& in_arg : const_inputs) { + cluster_inputs.push_back(in_arg); + } + + std::copy(external_output_args.begin(), external_output_args.end(), std::back_inserter(cluster_outputs)); + for (const auto& node_arg : graph_viewer.GetOutputs()) { + const auto& name = node_arg->Name(); + if (output_args.count(name) && !external_output_args.count(name)) { + cluster_outputs.push_back(name); + } + } +} + +static void AppendClusterToSubGraph(const std::vector& nodes, + const std::vector& inputs, + const std::vector& outputs, + std::vector>& result) { + static size_t op_counter = 0; + + auto meta_def = onnxruntime::make_unique(); + meta_def->name = "VitisAICustomOp_" + std::to_string(++op_counter); + meta_def->domain = kVitisAIDomain; + meta_def->since_version = 1; + meta_def->status = ONNX_NAMESPACE::EXPERIMENTAL; + meta_def->inputs = inputs; + meta_def->outputs = outputs; + + std::unique_ptr sub_graph = onnxruntime::make_unique(); + sub_graph->nodes = nodes; + sub_graph->SetMetaDef(meta_def); + result.push_back(onnxruntime::make_unique(std::move(sub_graph))); +} + + +std::vector> +VitisAIExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& kernel_registries) const { + ORT_UNUSED_PARAMETER(kernel_registries); + + std::vector> result; + + // Dump model Proto to file to pass it to pyxir + auto logger = *GetLogger(); + + const Graph& node_graph = graph.GetGraph(); + const std::string& name_ = node_graph.Name(); + onnxruntime::Model model{name_, true, ModelMetaData{}, PathString{}, + IOnnxRuntimeOpSchemaRegistryList{}, + node_graph.DomainToVersionMap(), + std::vector(), + logger}; + + ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); + model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + + *(model_proto.mutable_graph()) = node_graph.ToGraphProto(); + + std::istringstream model_stream{model_proto.SerializeAsString()}; + + // Transform ONNX into Pyxir XGraph data structure + XGraphHolder xg = pyxir::onnx::import_onnx_model(model_stream); + + // Annotate the subgraph layers in the XGraph that can be executed on the + // specified `backend_type_` target + pyxir::partition(xg, std::vector{backend_type_}, ""); + + // Next stuff + if (graph.IsSubgraph()) { + return result; + } + + // Need access to model_path_ + for (const auto& tensor : graph.GetAllInitializedTensors()) { + if (tensor.second->has_data_location() && tensor.second->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { + LOGS_DEFAULT(WARNING) << "VITIS-AI EP: Initializers with external data location are not currently supported"; + return result; + } + } + + std::unordered_set required_initializers; + const auto unsupported_nodes = GetUnsupportedNodeIndices(xg, backend_type_, graph, required_initializers); + + const auto clusters = GetPartitionedClusters(graph.GetNodesInTopologicalOrder(), unsupported_nodes); + + for (const auto& this_cluster : clusters) { + std::vector cluster_inputs, cluster_outputs; + GetInputsOutputsOfCluster(graph, this_cluster, required_initializers, cluster_inputs, cluster_outputs); + + if (!cluster_inputs.empty()) { + AppendClusterToSubGraph(this_cluster, cluster_inputs, cluster_outputs, result); + } + } + + return result; +} + +common::Status VitisAIExecutionProvider::Compile(const std::vector& fused_nodes, + std::vector& node_compute_funcs) { + for (const auto& fused_node : fused_nodes) + { + NodeComputeInfo compute_info; + compute_info.create_state_func = [this, fused_node, logger=GetLogger()](ComputeContext* context, FunctionState* state) { + auto* p = new vitisai_ep::VitisAICustomOp(context, fused_node, backend_type_, logger); + *state = p; + return 0; + }; + + compute_info.release_state_func = [](FunctionState state) { + if (state) + delete reinterpret_cast(state); + }; + + compute_info.compute_func = [](FunctionState state, const OrtApi* api, OrtKernelContext* context) { + onnxruntime::vitisai_ep::VitisAICustomOp* custom_op = reinterpret_cast(state); + return custom_op->Compute(api, context); + }; + + node_compute_funcs.push_back(compute_info); + } + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h new file mode 100644 index 0000000000..4450425b92 --- /dev/null +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h @@ -0,0 +1,38 @@ +// Copyright (c) Xilinx Inc. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/framework/execution_provider.h" +#include "core/platform/ort_mutex.h" + +namespace onnxruntime { + +// Information needed to construct execution providers. +struct VitisAIExecutionProviderInfo { + int device_id{0}; + std::string backend_type; +}; + +// Logical device representation. +class VitisAIExecutionProvider : public IExecutionProvider { + public: + explicit VitisAIExecutionProvider(const VitisAIExecutionProviderInfo& info); + ~VitisAIExecutionProvider() = default; + + std::vector> + GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& /*kernel_registries*/) const override; + + int GetDeviceId() const { return device_id_; } + + common::Status Compile(const std::vector& fused_nodes, + std::vector& node_compute_funcs) override; + + private: + std::string backend_type_; + int device_id_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc b/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc new file mode 100755 index 0000000000..58a8505be6 --- /dev/null +++ b/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc @@ -0,0 +1,41 @@ +// Copyright (c) Xilinx Inc. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/vitisai/vitisai_provider_factory.h" +#include +#include "vitisai_execution_provider.h" +#include "core/session/abi_session_options_impl.h" + +using namespace onnxruntime; + +namespace onnxruntime { + +struct VitisAIProviderFactory : IExecutionProviderFactory { + VitisAIProviderFactory(std::string&& backend_type, int device_id) + : backend_type_(std::move(backend_type)), device_id_(device_id) {} + ~VitisAIProviderFactory() = default; + + std::unique_ptr CreateProvider() override; + + private: + const std::string backend_type_; + int device_id_; +}; + +std::unique_ptr VitisAIProviderFactory::CreateProvider() { + VitisAIExecutionProviderInfo info; + info.backend_type = backend_type_; + info.device_id = device_id_; + return onnxruntime::make_unique(info); +} + +std::shared_ptr CreateExecutionProviderFactory_VITISAI(const char *backend_type, int device_id) { + return std::make_shared(backend_type, device_id); +} +} // namespace onnxruntime + +ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_VITISAI, _In_ OrtSessionOptions* options, _In_ const char* backend_type, int device_id) { + options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_VITISAI(backend_type, device_id)); + return nullptr; +} + diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index bafe3abb71..d6d64d56a3 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -81,6 +81,13 @@ #define BACKEND_NUPHAR "" #endif +#if USE_VITISAI +#define BACKEND_VITISAI "-VITISAI" +#include "core/providers/vitisai/vitisai_execution_provider.h" +#else +#define BACKEND_VITISAI "" +#endif + #if USE_OPENBLAS #define BACKEND_OPENBLAS "-OPENBLAS" #else @@ -113,6 +120,9 @@ std::string openvino_device; #include "core/providers/nuphar/nuphar_provider_factory.h" std::string nuphar_settings; #endif +#ifdef USE_VITISAI +#include "core/providers/vitisai/vitisai_provider_factory.h" +#endif namespace onnxruntime { std::shared_ptr CreateExecutionProviderFactory_CPU(int use_arena); @@ -124,6 +134,8 @@ std::shared_ptr CreateExecutionProviderFactory_Dnnl(i std::shared_ptr CreateExecutionProviderFactory_NGraph(const char* ng_backend_type); std::shared_ptr CreateExecutionProviderFactory_OpenVINO(const char* device); std::shared_ptr CreateExecutionProviderFactory_Nuphar(bool, const char*); +std::shared_ptr CreateExecutionProviderFactory_VITISAI(const char *backend_type, int device_id); + } // namespace onnxruntime #if defined(_MSC_VER) @@ -265,7 +277,7 @@ inline void RegisterExecutionProvider(InferenceSession* sess, onnxruntime::IExec const std::vector& GetAllProviders() { static std::vector all_providers = {kTensorrtExecutionProvider, kCudaExecutionProvider, kDnnlExecutionProvider, kNGraphExecutionProvider, kOpenVINOExecutionProvider, kNupharExecutionProvider, - kCpuExecutionProvider}; + kVitisAIExecutionProvider, kCpuExecutionProvider}; return all_providers; } @@ -289,6 +301,9 @@ const std::vector& GetAvailableProviders() { #endif #ifdef USE_NUPHAR available_providers.push_back(kNupharExecutionProvider); +#endif +#ifdef USE_VITISAI + available_providers.push_back(kVitisAIExecutionProvider); #endif return available_providers; }; @@ -328,6 +343,10 @@ void RegisterExecutionProviders(InferenceSession* sess, const std::vector