mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-23 22:13:38 +00:00
1942e40e05
### Description For fp16 Atype, the fallback operation is convert the data to fp32 and calculate. Added neon intrinsics version to speed up the conversion. Store address alignment and loop unrolling have insignificant impact on latency so they are omitted. |Benchmark | Time | CPU | |--------------|---------------------------------------------|--------------------| |M_ConvertF16ToF32/baseline/real_time | 1076961 ns | 1083398 ns | |M_ConvertF16ToF32/aligned:0/real_time | 46785 ns | 46516 ns | |M_ConvertF16ToF32/aligned:1/real_time | 46631 ns | 46391 ns | |M_ConvertF16ToF32_unroll2/aligned:0/real_time | 44074 ns | 44392 ns | |M_ConvertF16ToF32_unroll2/aligned:1/real_time | 44726 ns | 45226 ns | |M_ConvertF32ToF16/baseline/real_time | 520109 ns | 527329 ns | |M_ConvertF32ToF16/aligned:0/real_time | 73610 ns | 74015 ns | |M_ConvertF32ToF16/aligned:1/real_time | 71557 ns | 71525 ns | |M_ConvertF32ToF16_unroll2/aligned:0/real_time | 64227 ns | 63374 ns | |M_ConvertF32ToF16_unroll2/aligned:1/real_time | 67428 ns | 67989 ns | ### Motivation and Context speed up fallback implementation of Fp16 MatMulNBits |
||
|---|---|---|
| .. | ||
| external | ||
| patches | ||
| tensorboard | ||
| adjust_global_compile_flags.cmake | ||
| arm64x.cmake | ||
| CMakeLists.txt | ||
| CMakePresets.json | ||
| CMakeSettings.json | ||
| codeconv.runsettings | ||
| deps.txt | ||
| deps_update_and_upload.py | ||
| EnableVisualStudioCodeAnalysis.props | ||
| gdk_toolchain.cmake | ||
| Info.plist.in | ||
| libonnxruntime.pc.cmake.in | ||
| linux_arm32_crosscompile_toolchain.cmake | ||
| linux_arm64_crosscompile_toolchain.cmake | ||
| maccatalyst_prepare_objects_for_prelink.py | ||
| nuget_helpers.cmake | ||
| onnxruntime.cmake | ||
| onnxruntime_codegen_tvm.cmake | ||
| onnxruntime_common.cmake | ||
| onnxruntime_compile_triton_kernel.cmake | ||
| onnxruntime_config.h.in | ||
| onnxruntime_csharp.cmake | ||
| onnxruntime_flatbuffers.cmake | ||
| onnxruntime_framework.cmake | ||
| onnxruntime_framework.natvis | ||
| onnxruntime_fuzz_test.cmake | ||
| onnxruntime_graph.cmake | ||
| onnxruntime_ios.toolchain.cmake | ||
| onnxruntime_java.cmake | ||
| onnxruntime_java_unittests.cmake | ||
| onnxruntime_kernel_explorer.cmake | ||
| onnxruntime_mlas.cmake | ||
| onnxruntime_nodejs.cmake | ||
| onnxruntime_objectivec.cmake | ||
| onnxruntime_opschema_lib.cmake | ||
| onnxruntime_optimizer.cmake | ||
| onnxruntime_providers.cmake | ||
| onnxruntime_providers_acl.cmake | ||
| onnxruntime_providers_armnn.cmake | ||
| onnxruntime_providers_azure.cmake | ||
| onnxruntime_providers_cann.cmake | ||
| onnxruntime_providers_coreml.cmake | ||
| onnxruntime_providers_cpu.cmake | ||
| onnxruntime_providers_cuda.cmake | ||
| onnxruntime_providers_dml.cmake | ||
| onnxruntime_providers_dnnl.cmake | ||
| onnxruntime_providers_js.cmake | ||
| onnxruntime_providers_migraphx.cmake | ||
| onnxruntime_providers_nnapi.cmake | ||
| onnxruntime_providers_openvino.cmake | ||
| onnxruntime_providers_qnn.cmake | ||
| onnxruntime_providers_rknpu.cmake | ||
| onnxruntime_providers_rocm.cmake | ||
| onnxruntime_providers_tensorrt.cmake | ||
| onnxruntime_providers_tvm.cmake | ||
| onnxruntime_providers_vitisai.cmake | ||
| onnxruntime_providers_vsinpu.cmake | ||
| onnxruntime_providers_webnn.cmake | ||
| onnxruntime_providers_xnnpack.cmake | ||
| onnxruntime_python.cmake | ||
| onnxruntime_rocm_hipify.cmake | ||
| onnxruntime_session.cmake | ||
| onnxruntime_snpe_provider.cmake | ||
| onnxruntime_training.cmake | ||
| onnxruntime_unittests.cmake | ||
| onnxruntime_util.cmake | ||
| onnxruntime_visionos.toolchain.cmake | ||
| onnxruntime_webassembly.cmake | ||
| precompiled_header.cmake | ||
| riscv64.toolchain.cmake | ||
| Sdl.ruleset | ||
| set_winapi_family_desktop.h | ||
| target_delayload.cmake | ||
| uwp_stubs.h | ||
| vcpkg-configuration.json | ||
| vcpkg.json | ||
| wcos_rules_override.cmake | ||
| winml.cmake | ||
| winml_cppwinrt.cmake | ||
| winml_sdk_helpers.cmake | ||
| winml_unittests.cmake | ||