From 7cbcf5cd4b12fd264083a250409227e65961817f Mon Sep 17 00:00:00 2001 From: Chunye Wang Date: Mon, 19 Aug 2024 23:21:01 -0700 Subject: [PATCH] refactor VitisAI EP for opensource update to use integrated vaip update vaip as the single entry point for cmake add dummy vaip_xcompile_run. onnxruntime_vitisia_ep.dll is optional. vaip_xcompiler_compile maybe nullptr. update create_ep_context_nodes. --- cmake/deps.txt | 1 + .../external/onnxruntime_external_deps.cmake | 1 - cmake/onnxruntime_providers_vitisai.cmake | 24 +++++++- .../core/providers/vitisai/imp/global_api.cc | 55 ++++++++++++------- .../vitisai/include/vaip/vaip_ort_api.h | 7 ++- 5 files changed, 66 insertions(+), 22 deletions(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index d1a528bd6b..2eb78d5756 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -60,3 +60,4 @@ directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v cudnn_frontend;https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.7.0.zip;d0753d8d5b39947ca0729d7773cb84653a129eb1 dawn;https://github.com/google/dawn/archive/12a3b24c456cebd9fd11f23ac0164f78129b00c6.zip;ad428f6dc16f1336d584f7bad5714e1097dafc43 kleidiai;https://gitlab.arm.com/kleidi/kleidiai/-/archive/d15722976120710080ca098fe8ddabf4556cb40f/kleidiai-d15722976120710080ca098fe8ddabf4556cb40f.zip;d6c840d00c3b05aedf06e957ddaece1013d1f40b +vaip;https://gitenterprise.xilinx.com/chunywan/vaip.git;refactor-vaip diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 761ce47582..773a5ee460 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -198,7 +198,6 @@ FetchContent_Declare( PATCH_COMMAND ${ONNXRUNTIME_PROTOBUF_PATCH_COMMAND} FIND_PACKAGE_ARGS NAMES Protobuf protobuf ) - set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) #TODO: we'd better to turn the following option off. However, it will cause # ".\build.bat --config Debug --parallel --skip_submodule_sync --update" fail with an error message: diff --git a/cmake/onnxruntime_providers_vitisai.cmake b/cmake/onnxruntime_providers_vitisai.cmake index 561a323533..fa87cc4be3 100644 --- a/cmake/onnxruntime_providers_vitisai.cmake +++ b/cmake/onnxruntime_providers_vitisai.cmake @@ -1,5 +1,26 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. + FetchContent_Declare( + vaip + GIT_REPOSITORY ${DEP_URL_vaip} + GIT_TAG ${DEP_SHA1_vaip} + GIT_SUBMODULES_RECURSE FALSE + GIT_SHALLOW TRUE + OVERRIDE_FIND_PACKAGE + ) + set(WITH_XCOMPILER ON CACHE BOOL "enable XCOMPILER") + set(WITH_OPENSSL OFF CACHE BOOL "enable open ssl") + set(WITH_CPURUNNER ON CACHE BOOL "enable cpu runner") + set(BUILD_PYTHON_EXT OFF CACHE BOOL "enable python ext") + set(EN_LLM_DOD_OPS OFF CACHE BOOL "enable dd flow") + set(EN_VAIML OFF CACHE BOOL "enable vaiml flow") + set(TRIM_CONFIG OFF CACHE BOOL "enable config trimming") + set(ENABLE_VITIS_AI_CUSTOM_OP OFF "enable vitis ai custom op") + set(PACK_XCLBIN_PATH "" CACHE STRING "list of xclbin files") + set(ENABLE_BUILD_VOE_WHEEL OFF CACHE BOOL "internal used only" FORCE) + set(INSTALL_USER ON CACHE BOOL "internal used only" FORCE) + set(ENABLE_XRT_SHARED_CONTEXT ON CACHE BOOL "internal used only" FORCE) + find_package(vaip) if ("${GIT_COMMIT_ID}" STREQUAL "") execute_process( @@ -21,10 +42,11 @@ source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vitisai_cc_srcs}) onnxruntime_add_shared_library(onnxruntime_providers_vitisai ${onnxruntime_providers_vitisai_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_vitisai ${ONNXRUNTIME_PROVIDERS_SHARED} ${GSL_TARGET} safeint_interface flatbuffers::flatbuffers) - target_link_libraries(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_PROVIDERS_SHARED}) + target_link_libraries(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_PROVIDERS_SHARED} onnxruntime_vitisai_ep::onnxruntime_vitisai_ep) if(MSVC) onnxruntime_add_include_to_target(onnxruntime_providers_vitisai dbghelp) set_property(TARGET onnxruntime_providers_vitisai APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${ONNXRUNTIME_ROOT}/core/providers/vitisai/symbols.def") + target_sources(onnxruntime_providers_vitisai PRIVATE ${vaip_BINARY_DIR}/onnxruntime_vitisai_ep/onnxruntime_vitisai_ep.def) else(MSVC) set_property(TARGET onnxruntime_providers_vitisai APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/core/providers/vitisai/version_script.lds -Xlinker --gc-sections") endif(MSVC) diff --git a/onnxruntime/core/providers/vitisai/imp/global_api.cc b/onnxruntime/core/providers/vitisai/imp/global_api.cc index 8111ee3c1f..c017ce922d 100644 --- a/onnxruntime/core/providers/vitisai/imp/global_api.cc +++ b/onnxruntime/core/providers/vitisai/imp/global_api.cc @@ -42,6 +42,16 @@ using namespace onnxruntime; #define LIBRARY_EXTENSION ".so" #endif +namespace vaip_core { + +void initialize_onnxruntime_vitisai_ep(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); +uint32_t vaip_get_version(); +extern "C" int create_ep_context_nodes( + const std::vector>& eps, + vaip_core::DllSafe>* ret_value); +std::vector>* compile_onnx_model_with_options( + const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options); +} // namespace vaip_core vaip_core::OrtApiForVaip* create_org_api_hook(); struct OrtVitisAIEpAPI { void (*initialize_onnxruntime_vitisai_ep)(vaip_core::OrtApiForVaip* api, std::vector& ret_domain); @@ -50,7 +60,7 @@ struct OrtVitisAIEpAPI { std::vector>* (*compile_onnx_model_vitisai_ep_with_error_handling)( const std::string& model_path, const onnxruntime::Graph& graph, const onnxruntime::ProviderOptions& options, void* status, vaip_core::error_report_func func); uint32_t (*vaip_get_version)(); - void (*create_ep_context_nodes)( + int (*create_ep_context_nodes)( const std::vector>& eps, vaip_core::DllSafe>* ret_value) = nullptr; int (*vitisai_ep_on_run_start)( @@ -63,34 +73,37 @@ struct OrtVitisAIEpAPI { void (*profiler_collect)( std::vector& api_events, std::vector& kernel_events); + int (*vaip_xcompiler_compile)(const char* input_xmodel, + size_t input_xmodel_size, + const char* config_xmodel, + size_t config_xmodel_size, void* state, + char* (*allocator)(void*, size_t)) = nullptr; void Ensure() { - if (handle_) - return; + this->initialize_onnxruntime_vitisai_ep = vaip_core::initialize_onnxruntime_vitisai_ep; + this->compile_onnx_model_with_options = vaip_core::compile_onnx_model_with_options; + this->create_ep_context_nodes = vaip_core::create_ep_context_nodes; + this->vaip_get_version = vaip_core::vaip_get_version; + // auto& env = Provider_GetHost()->Env__Default(); + auto& logger = *Provider_GetHost()->LoggingManager_GetDefaultLogger(); + #ifdef _WIN32 // this dll is already linked to the executable, normally a test program handle_ = reinterpret_cast(GetModuleHandle(TEXT("onnxruntime_vitisai_ep.dll"))); + auto status = Status::OK(); if (!handle_) { auto full_path = env.GetRuntimePath() + PathString(LIBRARY_PREFIX ORT_TSTR("onnxruntime_vitisai_ep") LIBRARY_EXTENSION); - ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_)); + status = env.LoadDynamicLibrary(full_path, true, &handle_); + if (!status.IsOK()) { + LOGS(logger, VERBOSE) << "cannot load onnxruntime_vitisai_ep.dll, can only deploy ep_context onnx model."; + return; + } } #else auto full_path = env.GetRuntimePath() + PathString(LIBRARY_PREFIX ORT_TSTR("onnxruntime_vitisai_ep") LIBRARY_EXTENSION); ORT_THROW_IF_ERROR(env.LoadDynamicLibrary(full_path, true, &handle_)); #endif - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "initialize_onnxruntime_vitisai_ep", (void**)&initialize_onnxruntime_vitisai_ep)); - auto status1 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_error_handling", (void**)&compile_onnx_model_vitisai_ep_with_error_handling); - auto status2 = env.GetSymbolFromLibrary(handle_, "compile_onnx_model_vitisai_ep_with_options", (void**)&compile_onnx_model_with_options); - if ((!status1.IsOK()) && (!status2.IsOK())) { - ::onnxruntime::LogRuntimeError(0, status2, __FILE__, static_cast(__FUNCTION__), __LINE__); - ORT_THROW(status2); - } - std::ignore = env.GetSymbolFromLibrary(handle_, "vaip_get_version", - (void**)&vaip_get_version); - std::ignore = env.GetSymbolFromLibrary(handle_, "profiler_collect", (void**)&profiler_collect); - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "create_ep_context_nodes", (void**)&create_ep_context_nodes)); - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vitisai_ep_on_run_start", (void**)&vitisai_ep_on_run_start)); - ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vitisai_ep_set_ep_dynamic_options", (void**)&vitisai_ep_set_ep_dynamic_options)); + ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(handle_, "vaip_xcompiler_compile", (void**)&vaip_xcompiler_compile)); } void Clear() { if (handle_) { @@ -107,9 +120,10 @@ struct OrtVitisAIEpAPI { static OrtVitisAIEpAPI s_library_vitisaiep; static std::shared_ptr s_kernel_registry_vitisaiep; -static std::vector s_domains_vitisaiep; static vaip_core::OrtApiForVaip the_global_api; -std::shared_ptr get_kernel_registry_vitisaiep() { return s_kernel_registry_vitisaiep; } +g + std::shared_ptr + get_kernel_registry_vitisaiep() { return s_kernel_registry_vitisaiep; } const std::vector& get_domains_vitisaiep() { return s_domains_vitisaiep; } void profiler_collect( @@ -484,6 +498,7 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { } }; the_global_api.node_arg_external_location = vaip::node_arg_external_location; + the_global_api.model_to_proto = [](onnxruntime::Model& model) { return model.ToProto().release(); }; the_global_api.model_proto_serialize_as_string = [](ONNX_NAMESPACE::ModelProto& model_proto) { return vaip_core::DllSafe(model_proto.SerializeAsString()); @@ -508,6 +523,8 @@ vaip_core::OrtApiForVaip* create_org_api_hook() { } else { return &the_global_api; } + the_global_api.vaip_xcompiler_compile = s_library_vitisaiep.vaip_xcompiler_compile; + return &the_global_api; } void deinitialize_vitisai_ep() { diff --git a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h index 6a51ef8622..2494558b1a 100644 --- a/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h +++ b/onnxruntime/core/providers/vitisai/include/vaip/vaip_ort_api.h @@ -235,7 +235,7 @@ struct OrtApiForVaip { DllSafe (*model_proto_serialize_as_string)(ModelProto& model_proto); // [96] void (*model_proto_delete)(ModelProto* p); // [97] DllSafe (*attr_proto_release_string)(AttributeProto* attr); // [98] - bool (*is_profiling_enabled)(void* session_options); // [99] // [98] + bool (*is_profiling_enabled)(void* session_options); // [99] TensorProto* (*tensor_proto_new_i4)(const std::string& name, const std::vector& shape, const std::vector& data); // [100] @@ -243,6 +243,11 @@ struct OrtApiForVaip { const std::vector& shape, const std::vector& data); // [101] void (*graph_remove_initialized_tensor)(Graph& graph, const std::string& tensor_name); // [102] + int (*vaip_xcompiler_compile)(const char* input_xmodel, + size_t input_xmodel_size, + const char* config_xmodel, + size_t config_xmodel_size, void* state, + char* (*allocator)(void*, size_t)); // [103] }; #ifndef USE_VITISAI