mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-26 19:52:38 +00:00
Detecting ARM64 CPU core micro-architectures in Windows (#11145)
Some micro-architectures of power efficient cores in ARMv8 system have narrow 64b load/store resources, which require specialized computing kernels in MLAS. We leverage pytorch CPUinfo package for detecting these cores. Unfortunately CPUinfo package does not work on Windows. This commit implements ARM64 micro-architecture detection.
This commit is contained in:
Chen Fu
GitHub
parent
ddb17294b2
commit
0d0edc071f
7 changed files with 867 additions and 188 deletions
|
|
@ -1,20 +1,19 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/common/cpuid_info.h"
|
||||
#include "core/common/logging/logging.h"
|
||||
#include "core/common/logging/severity.h"
|
||||
|
||||
#if defined(CPUIDINFO_ARCH_X86)
|
||||
#include <memory>
|
||||
#if defined(_MSC_VER)
|
||||
#include <intrin.h>
|
||||
#elif defined(__GNUC__)
|
||||
#include <cpuid.h>
|
||||
#endif
|
||||
#ifdef __linux__
|
||||
|
||||
#include <unistd.h>
|
||||
#include <sys/syscall.h>
|
||||
#if !defined(__NR_getcpu)
|
||||
#include <asm-generic/unistd.h>
|
||||
#endif
|
||||
|
||||
#if defined(CPUIDINFO_ARCH_ARM) && defined(CPUINFO_SUPPORTED) && defined(__linux__)
|
||||
#if defined(CPUIDINFO_ARCH_ARM)
|
||||
|
||||
#include <sys/auxv.h>
|
||||
#include <asm/hwcap.h>
|
||||
// N.B. Support building with older versions of asm/hwcap.h that do not define
|
||||
|
|
@ -23,20 +22,40 @@
|
|||
#define HWCAP_ASIMDDP (1 << 20)
|
||||
#endif
|
||||
|
||||
#endif
|
||||
#endif // ARM
|
||||
|
||||
#include <mutex>
|
||||
#endif // Linux
|
||||
|
||||
#if _WIN32
|
||||
|
||||
#include "Windows.h"
|
||||
|
||||
#define HAS_WINDOWS_DESKTOP WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
|
||||
|
||||
#ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE
|
||||
#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
|
||||
#endif
|
||||
#if (defined(CPUIDINFO_ARCH_X86) || defined(CPUIDINFO_ARCH_ARM)) && defined(CPUINFO_SUPPORTED) && (!_WIN32 || defined(HAS_WINDOWS_DESKTOP))
|
||||
|
||||
#endif // _WIN32
|
||||
|
||||
#if defined(CPUINFO_SUPPORTED)
|
||||
#include <cpuinfo.h>
|
||||
#endif
|
||||
#else
|
||||
#include "core/common/cpuid_uarch.h"
|
||||
#endif // CPUINFO_SUPPORTED
|
||||
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
#if defined(CPUIDINFO_ARCH_X86)
|
||||
#ifdef CPUIDINFO_ARCH_X86
|
||||
|
||||
#include <memory>
|
||||
#if defined(_MSC_VER)
|
||||
#include <intrin.h>
|
||||
#elif defined(__GNUC__)
|
||||
#include <cpuid.h>
|
||||
#endif
|
||||
|
||||
static inline void GetCPUID(int function_id, int data[4]) { // NOLINT
|
||||
#if defined(_MSC_VER)
|
||||
__cpuid(reinterpret_cast<int*>(data), function_id);
|
||||
|
|
@ -56,83 +75,158 @@ static inline int XGETBV() {
|
|||
return eax;
|
||||
#endif
|
||||
}
|
||||
#endif // CPUIDINFO_ARCH_X86
|
||||
|
||||
void CPUIDInfo::X86Init() {
|
||||
int data[4] = {-1};
|
||||
GetCPUID(0, data);
|
||||
|
||||
CPUIDInfo::CPUIDInfo() {
|
||||
#if (defined(CPUIDINFO_ARCH_X86) || defined(CPUIDINFO_ARCH_ARM)) && defined(CPUINFO_SUPPORTED)
|
||||
int num_IDs = data[0];
|
||||
if (num_IDs >= 1) {
|
||||
GetCPUID(1, data);
|
||||
if (data[2] & (1 << 27)) {
|
||||
constexpr int AVX_MASK = 0x6;
|
||||
constexpr int AVX512_MASK = 0xE6;
|
||||
int value = XGETBV();
|
||||
bool has_sse2 = (data[3] & (1 << 26));
|
||||
has_sse3_ = (data[2] & 0x1);
|
||||
has_sse4_1_ = (data[2] & (1 << 19));
|
||||
bool has_ssse3 = (data[2] & (1 << 9));
|
||||
has_avx_ = has_sse2 && has_ssse3 && (data[2] & (1 << 28)) && ((value & AVX_MASK) == AVX_MASK);
|
||||
bool has_avx512 = (value & AVX512_MASK) == AVX512_MASK;
|
||||
has_f16c_ = has_avx_ && (data[2] & (1 << 29)) && (data[3] & (1 << 26));
|
||||
|
||||
if (num_IDs >= 7) {
|
||||
GetCPUID(7, data);
|
||||
has_avx2_ = has_avx_ && (data[1] & (1 << 5));
|
||||
has_avx512f_ = has_avx512 && (data[1] & (1 << 16));
|
||||
// Add check for AVX512 Skylake since tensorization GEMM need intrinsics from avx512bw/avx512dq.
|
||||
// avx512_skylake = avx512f | avx512vl | avx512cd | avx512bw | avx512dq
|
||||
has_avx512_skylake_ = has_avx512 && (data[1] & ((1 << 16) | (1 << 17) | (1 << 28) | (1 << 30) | (1 << 31)));
|
||||
is_hybrid_ = (data[3] & (1 << 15));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* CPUIDINFO_ARCH_X86 */
|
||||
|
||||
#if defined(CPUIDINFO_ARCH_ARM)
|
||||
#ifdef __linux__
|
||||
|
||||
void CPUIDInfo::ArmLinuxInit() {
|
||||
// Pytorch CPUINFO only works on ARM linux or android
|
||||
// Assuming no hyper-threading, no NUMA groups
|
||||
#ifdef CPUINFO_SUPPORTED
|
||||
pytorch_cpuinfo_init_ = cpuinfo_initialize();
|
||||
if (!pytorch_cpuinfo_init_) {
|
||||
LOGS_DEFAULT(WARNING) << "Failed to init pytorch cpuinfo library, may cause CPU EP performance degradation due to undetected CPU features.";
|
||||
return;
|
||||
}
|
||||
#else
|
||||
pytorch_cpuinfo_init_ = false;
|
||||
#endif
|
||||
|
||||
#if defined(CPUIDINFO_ARCH_X86)
|
||||
int data[4] = {-1};
|
||||
GetCPUID(0, data);
|
||||
|
||||
int num_IDs = data[0];
|
||||
if (num_IDs >= 1) {
|
||||
GetCPUID(1, data);
|
||||
if (data[2] & (1 << 27)) {
|
||||
constexpr int AVX_MASK = 0x6;
|
||||
constexpr int AVX512_MASK = 0xE6;
|
||||
int value = XGETBV();
|
||||
bool has_sse2 = (data[3] & (1 << 26));
|
||||
has_sse3_ = (data[2] & 0x1);
|
||||
has_sse4_1_ = (data[2] & (1 << 19));
|
||||
bool has_ssse3 = (data[2] & (1 << 9));
|
||||
has_avx_ = has_sse2 && has_ssse3 && (data[2] & (1 << 28)) && ((value & AVX_MASK) == AVX_MASK);
|
||||
bool has_avx512 = (value & AVX512_MASK) == AVX512_MASK;
|
||||
has_f16c_ = has_avx_ && (data[2] & (1 << 29)) && (data[3] & (1 << 26));
|
||||
|
||||
if (num_IDs >= 7) {
|
||||
GetCPUID(7, data);
|
||||
has_avx2_ = has_avx_ && (data[1] & (1 << 5));
|
||||
has_avx512f_ = has_avx512 && (data[1] & (1 << 16));
|
||||
// Add check for AVX512 Skylake since tensorization GEMM need intrinsics from avx512bw/avx512dq.
|
||||
// avx512_skylake = avx512f | avx512vl | avx512cd | avx512bw | avx512dq
|
||||
has_avx512_skylake_ = has_avx512 && (data[1] & ((1 << 16) | (1 << 17) | (1 << 28) | (1 << 30) | (1 << 31)));
|
||||
is_hybrid_ = (data[3] & (1 << 15));
|
||||
}
|
||||
if (pytorch_cpuinfo_init_) {
|
||||
is_hybrid_ = cpuinfo_get_uarchs_count() > 1;
|
||||
has_arm_neon_dot_ = cpuinfo_has_arm_neon_dot();
|
||||
const uint32_t core_cnt = cpuinfo_get_cores_count();
|
||||
core_uarchs_.resize(core_cnt, cpuinfo_uarch_unknown);
|
||||
is_armv8_narrow_ld_.resize(core_cnt, false);
|
||||
for (uint32_t c = 0; c < core_cnt; c++) {
|
||||
const struct cpuinfo_processor* proc = cpuinfo_get_processor(c);
|
||||
if (proc == nullptr) {
|
||||
continue;
|
||||
}
|
||||
const struct cpuinfo_core* corep = proc->core;
|
||||
if (corep == nullptr) {
|
||||
continue;
|
||||
}
|
||||
auto coreid = proc->linux_id;
|
||||
auto uarch = corep->uarch;
|
||||
core_uarchs_[coreid] = uarch;
|
||||
if (uarch == cpuinfo_uarch_cortex_a53 || uarch == cpuinfo_uarch_cortex_a55r0 ||
|
||||
uarch == cpuinfo_uarch_cortex_a55) {
|
||||
is_armv8_narrow_ld_[coreid] = true;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CPUIDINFO_ARCH_ARM)
|
||||
#ifdef HWCAP_ASIMDDP
|
||||
|
||||
// only works on ARM linux or android, does not work on Windows
|
||||
if (pytorch_cpuinfo_init_) {
|
||||
is_hybrid_ = cpuinfo_get_uarchs_count() > 1;
|
||||
has_arm_neon_dot_ = cpuinfo_has_arm_neon_dot();
|
||||
} else {
|
||||
has_arm_neon_dot_ = ((getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0);
|
||||
}
|
||||
} else {
|
||||
has_arm_neon_dot_ = ((getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0);
|
||||
}
|
||||
}
|
||||
|
||||
#elif defined(_WIN32)
|
||||
// TODO implement hardware feature detection in windows.
|
||||
is_hybrid_ = true;
|
||||
#endif
|
||||
#endif
|
||||
|
||||
void CPUIDInfo::ArmWindowsInit() {
|
||||
|
||||
#pragma region Application Family or OneCore Family
|
||||
#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM)
|
||||
// Read MIDR from windows registry
|
||||
// TODO!! Don't support multiple processor group yet!!
|
||||
constexpr int MAX_CORES = 64;
|
||||
constexpr int MAX_VALUE_NAME = 4096;
|
||||
|
||||
TCHAR midrKey[MAX_VALUE_NAME] = ""; // buffer for processor registry name
|
||||
uint32_t lastUarch = cpuinfo_uarch_unknown;
|
||||
for (int i = 0; i < MAX_CORES - 1; i++) {
|
||||
snprintf(midrKey, MAX_VALUE_NAME, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\%d", i);
|
||||
uint64_t midrVal;
|
||||
unsigned long midrSize = sizeof(uint64_t);
|
||||
|
||||
/*
|
||||
* ARM lists for each coprocessor register 5 fields: op0/op1/CRn/CRm/op2.
|
||||
* You need to put those numbers through the ARM64_SYSREG macro:
|
||||
*
|
||||
* #define ARM64_SYSREG(op0, op1, crn, crm, op2) \
|
||||
* (((op0 & 1) << 14) | \
|
||||
* ((op1 & 7) << 11) | \
|
||||
* ((crn & 15) << 7) | \
|
||||
* ((crm & 15) << 3) | \
|
||||
* ((op2 & 7) << 0))
|
||||
*
|
||||
* For the CP value of MIDR, op0 = 3 and the others are all = 0, so we come up with 0x4000,
|
||||
*/
|
||||
auto retCode = ::RegGetValue(HKEY_LOCAL_MACHINE, midrKey, "CP 4000", RRF_RT_REG_QWORD, nullptr, &midrVal, &midrSize);
|
||||
if (retCode != ERROR_SUCCESS) {
|
||||
break;
|
||||
}
|
||||
uint32_t uarch = cpuinfo_uarch_unknown;
|
||||
decodeMIDR((uint32_t)midrVal, &uarch);
|
||||
core_uarchs_.push_back(uarch);
|
||||
if (uarch == cpuinfo_uarch_cortex_a53 || uarch == cpuinfo_uarch_cortex_a55r0 ||
|
||||
uarch == cpuinfo_uarch_cortex_a55) {
|
||||
is_armv8_narrow_ld_.push_back(true);
|
||||
} else {
|
||||
is_armv8_narrow_ld_.push_back(false);
|
||||
}
|
||||
|
||||
if (i == 0) {
|
||||
lastUarch = uarch;
|
||||
} else if (lastUarch != uarch) {
|
||||
is_hybrid_ = true;
|
||||
lastUarch = uarch;
|
||||
}
|
||||
}
|
||||
#endif /* Application Family or OneCore Family */
|
||||
|
||||
has_arm_neon_dot_ = (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0);
|
||||
}
|
||||
|
||||
int32_t CPUIDInfo::GetCurrentUarch() const {
|
||||
#if (defined(CPUIDINFO_ARCH_X86) || defined(CPUIDINFO_ARCH_ARM)) && defined(CPUINFO_SUPPORTED)
|
||||
if (!pytorch_cpuinfo_init_) {
|
||||
return -1;
|
||||
}
|
||||
const auto uarchIdx = cpuinfo_get_current_uarch_index();
|
||||
const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(uarchIdx);
|
||||
if (uarch_info == NULL) {
|
||||
return -1;
|
||||
}
|
||||
return uarch_info->uarch;
|
||||
#endif /* (arm or arm64) and windows */
|
||||
#endif /* arm or arm64*/
|
||||
|
||||
uint32_t CPUIDInfo::GetCurrentCoreIdx() const {
|
||||
#ifdef _WIN32
|
||||
return GetCurrentProcessorNumber();
|
||||
#elif defined(__linux__)
|
||||
uint32_t coreIdx = 0xFFFFFFFF;
|
||||
if (syscall(__NR_getcpu, &coreIdx, NULL, NULL) != 0) {
|
||||
// failed to detect current core id. give up
|
||||
return 0xFFFFFFFF;
|
||||
}
|
||||
return coreIdx;
|
||||
#else
|
||||
return -1;
|
||||
return 0xFFFFFFFF; // don't know how to get core index
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace onnxruntime
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -11,7 +11,7 @@
|
|||
|
||||
#if defined(_M_ARM64) || defined(__aarch64__) || defined(_M_ARM) || defined(__arm__)
|
||||
#define CPUIDINFO_ARCH_ARM
|
||||
#endif
|
||||
#endif // ARM or ARM64
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
@ -34,13 +34,76 @@ class CPUIDInfo {
|
|||
// ARM
|
||||
bool HasArmNeonDot() const { return has_arm_neon_dot_; }
|
||||
|
||||
uint32_t GetCurrentCoreIdx() const;
|
||||
|
||||
/**
|
||||
* @return CPU core micro-architecture running the current thread
|
||||
*/
|
||||
int32_t GetCurrentUarch() const;
|
||||
int32_t GetCurrentUarch() const {
|
||||
if (core_uarchs_.empty()) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint32_t coreIdx = GetCurrentCoreIdx();
|
||||
if (coreIdx >= core_uarchs_.size()) {
|
||||
return -1;
|
||||
}
|
||||
return core_uarchs_[coreIdx];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return CPU core micro-architecture
|
||||
*/
|
||||
int32_t GetCoreUarch(uint32_t coreId) const {
|
||||
if (coreId >= core_uarchs_.size()) {
|
||||
return -1;
|
||||
}
|
||||
return core_uarchs_[coreId];
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Some ARMv8 power efficient core has narrower 64b load/store
|
||||
* that needs specialized optimiztion in kernels
|
||||
* @return whether the indicated core has narrower load/store device
|
||||
*/
|
||||
bool IsCoreArmv8NarrowLd(uint32_t coreId) const {
|
||||
if (coreId >= is_armv8_narrow_ld_.size()) {
|
||||
return false;
|
||||
}
|
||||
return is_armv8_narrow_ld_[coreId];
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Some ARMv8 power efficient core has narrower 64b load/store
|
||||
* that needs specialized optimiztion in kernels
|
||||
* @return whether the current core has narrower load/store device
|
||||
*/
|
||||
bool IsCurrentCoreArmv8NarrowLd() const {
|
||||
if (is_armv8_narrow_ld_.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t coreIdx = GetCurrentCoreIdx();
|
||||
if (coreIdx >= is_armv8_narrow_ld_.size()) {
|
||||
return false;
|
||||
}
|
||||
return is_armv8_narrow_ld_[coreIdx];
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
CPUIDInfo();
|
||||
CPUIDInfo() {
|
||||
#ifdef CPUIDINFO_ARCH_X86
|
||||
X86Init();
|
||||
#elif defined(CPUIDINFO_ARCH_ARM)
|
||||
#ifdef __linux__
|
||||
ArmLinuxInit();
|
||||
#elif defined(_WIN32)
|
||||
ArmWindowsInit();
|
||||
#endif /* (arm or arm64) and windows */
|
||||
#endif
|
||||
|
||||
}
|
||||
bool has_avx_{false};
|
||||
bool has_avx2_{false};
|
||||
bool has_avx512f_{false};
|
||||
|
|
@ -50,10 +113,32 @@ class CPUIDInfo {
|
|||
bool has_sse4_1_{false};
|
||||
bool is_hybrid_{false};
|
||||
|
||||
#if (defined(CPUIDINFO_ARCH_X86) || defined(CPUIDINFO_ARCH_ARM)) && defined(CPUINFO_SUPPORTED)
|
||||
bool pytorch_cpuinfo_init_{false};
|
||||
#endif
|
||||
std::vector<uint32_t> core_uarchs_; // micro-arch of each core
|
||||
|
||||
// In ARMv8 systems, some power efficient cores has narrower
|
||||
// 64b load/store devices. It takes longer for them to load
|
||||
// 128b vectore registers.
|
||||
std::vector<bool> is_armv8_narrow_ld_;
|
||||
|
||||
bool has_arm_neon_dot_{false};
|
||||
|
||||
#ifdef CPUIDINFO_ARCH_X86
|
||||
|
||||
void X86Init();
|
||||
|
||||
#elif defined(CPUIDINFO_ARCH_ARM)
|
||||
#ifdef __linux__
|
||||
|
||||
bool pytorch_cpuinfo_init_{false};
|
||||
void ArmLinuxInit();
|
||||
|
||||
#elif defined(_WIN32)
|
||||
|
||||
void ArmWindowsInit();
|
||||
|
||||
#endif /* (arm or arm64) and windows */
|
||||
#endif
|
||||
|
||||
};
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
533
onnxruntime/core/common/cpuid_uarch.h
Normal file
533
onnxruntime/core/common/cpuid_uarch.h
Normal file
|
|
@ -0,0 +1,533 @@
|
|||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
cpuid_uarch.h
|
||||
|
||||
Abstract:
|
||||
ARM Processor microarchitecture descriptions and utilities
|
||||
|
||||
Processors with different microarchitectures often have different
|
||||
instruction performance characteristics, and may have dramatically
|
||||
different pipeline organization.
|
||||
|
||||
We leverage Pytorch CPUinfo (https://github.com/pytorch/cpuinfo) for
|
||||
hardware feature detection. Unfortunately CPUinfo package only works
|
||||
on a few platforms but we need to support more. This ugly hack is
|
||||
here to bridge the gap.
|
||||
|
||||
--*/
|
||||
|
||||
enum CPUIDINFOuarch {
|
||||
/** Microarchitecture is unknown, or the library failed to get information about the microarchitecture from OS */
|
||||
cpuinfo_uarch_unknown = 0,
|
||||
|
||||
/** Intel/Marvell XScale series. */
|
||||
cpuinfo_uarch_xscale = 0x00100600,
|
||||
|
||||
/** ARM7 series. */
|
||||
cpuinfo_uarch_arm7 = 0x00300100,
|
||||
/** ARM9 series. */
|
||||
cpuinfo_uarch_arm9 = 0x00300101,
|
||||
/** ARM 1136, ARM 1156, ARM 1176, or ARM 11MPCore. */
|
||||
cpuinfo_uarch_arm11 = 0x00300102,
|
||||
|
||||
/** ARM Cortex-A5. */
|
||||
cpuinfo_uarch_cortex_a5 = 0x00300205,
|
||||
/** ARM Cortex-A7. */
|
||||
cpuinfo_uarch_cortex_a7 = 0x00300207,
|
||||
/** ARM Cortex-A8. */
|
||||
cpuinfo_uarch_cortex_a8 = 0x00300208,
|
||||
/** ARM Cortex-A9. */
|
||||
cpuinfo_uarch_cortex_a9 = 0x00300209,
|
||||
/** ARM Cortex-A12. */
|
||||
cpuinfo_uarch_cortex_a12 = 0x00300212,
|
||||
/** ARM Cortex-A15. */
|
||||
cpuinfo_uarch_cortex_a15 = 0x00300215,
|
||||
/** ARM Cortex-A17. */
|
||||
cpuinfo_uarch_cortex_a17 = 0x00300217,
|
||||
|
||||
/** ARM Cortex-A32. */
|
||||
cpuinfo_uarch_cortex_a32 = 0x00300332,
|
||||
/** ARM Cortex-A35. */
|
||||
cpuinfo_uarch_cortex_a35 = 0x00300335,
|
||||
/** ARM Cortex-A53. */
|
||||
cpuinfo_uarch_cortex_a53 = 0x00300353,
|
||||
/** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities compared to revision 1+). */
|
||||
cpuinfo_uarch_cortex_a55r0 = 0x00300354,
|
||||
/** ARM Cortex-A55. */
|
||||
cpuinfo_uarch_cortex_a55 = 0x00300355,
|
||||
/** ARM Cortex-A57. */
|
||||
cpuinfo_uarch_cortex_a57 = 0x00300357,
|
||||
/** ARM Cortex-A65. */
|
||||
cpuinfo_uarch_cortex_a65 = 0x00300365,
|
||||
/** ARM Cortex-A72. */
|
||||
cpuinfo_uarch_cortex_a72 = 0x00300372,
|
||||
/** ARM Cortex-A73. */
|
||||
cpuinfo_uarch_cortex_a73 = 0x00300373,
|
||||
/** ARM Cortex-A75. */
|
||||
cpuinfo_uarch_cortex_a75 = 0x00300375,
|
||||
/** ARM Cortex-A76. */
|
||||
cpuinfo_uarch_cortex_a76 = 0x00300376,
|
||||
/** ARM Cortex-A77. */
|
||||
cpuinfo_uarch_cortex_a77 = 0x00300377,
|
||||
/** ARM Cortex-A78. */
|
||||
cpuinfo_uarch_cortex_a78 = 0x00300378,
|
||||
|
||||
/** ARM Neoverse N1. */
|
||||
cpuinfo_uarch_neoverse_n1 = 0x00300400,
|
||||
/** ARM Neoverse E1. */
|
||||
cpuinfo_uarch_neoverse_e1 = 0x00300401,
|
||||
|
||||
/** ARM Cortex-X1. */
|
||||
cpuinfo_uarch_cortex_x1 = 0x00300500,
|
||||
|
||||
/** Qualcomm Scorpion. */
|
||||
cpuinfo_uarch_scorpion = 0x00400100,
|
||||
/** Qualcomm Krait. */
|
||||
cpuinfo_uarch_krait = 0x00400101,
|
||||
/** Qualcomm Kryo. */
|
||||
cpuinfo_uarch_kryo = 0x00400102,
|
||||
/** Qualcomm Falkor. */
|
||||
cpuinfo_uarch_falkor = 0x00400103,
|
||||
/** Qualcomm Saphira. */
|
||||
cpuinfo_uarch_saphira = 0x00400104,
|
||||
|
||||
/** Nvidia Denver. */
|
||||
cpuinfo_uarch_denver = 0x00500100,
|
||||
/** Nvidia Denver 2. */
|
||||
cpuinfo_uarch_denver2 = 0x00500101,
|
||||
/** Nvidia Carmel. */
|
||||
cpuinfo_uarch_carmel = 0x00500102,
|
||||
|
||||
/** Samsung Exynos M1 (Exynos 8890 big cores). */
|
||||
cpuinfo_uarch_exynos_m1 = 0x00600100,
|
||||
/** Samsung Exynos M2 (Exynos 8895 big cores). */
|
||||
cpuinfo_uarch_exynos_m2 = 0x00600101,
|
||||
/** Samsung Exynos M3 (Exynos 9810 big cores). */
|
||||
cpuinfo_uarch_exynos_m3 = 0x00600102,
|
||||
/** Samsung Exynos M4 (Exynos 9820 big cores). */
|
||||
cpuinfo_uarch_exynos_m4 = 0x00600103,
|
||||
/** Samsung Exynos M5 (Exynos 9830 big cores). */
|
||||
cpuinfo_uarch_exynos_m5 = 0x00600104,
|
||||
|
||||
/* Deprecated synonym for Cortex-A76 */
|
||||
cpuinfo_uarch_cortex_a76ae = 0x00300376,
|
||||
/* Deprecated names for Exynos. */
|
||||
cpuinfo_uarch_mongoose_m1 = 0x00600100,
|
||||
cpuinfo_uarch_mongoose_m2 = 0x00600101,
|
||||
cpuinfo_uarch_meerkat_m3 = 0x00600102,
|
||||
cpuinfo_uarch_meerkat_m4 = 0x00600103,
|
||||
|
||||
/** Apple A6 and A6X processors. */
|
||||
cpuinfo_uarch_swift = 0x00700100,
|
||||
/** Apple A7 processor. */
|
||||
cpuinfo_uarch_cyclone = 0x00700101,
|
||||
/** Apple A8 and A8X processor. */
|
||||
cpuinfo_uarch_typhoon = 0x00700102,
|
||||
/** Apple A9 and A9X processor. */
|
||||
cpuinfo_uarch_twister = 0x00700103,
|
||||
/** Apple A10 and A10X processor. */
|
||||
cpuinfo_uarch_hurricane = 0x00700104,
|
||||
/** Apple A11 processor (big cores). */
|
||||
cpuinfo_uarch_monsoon = 0x00700105,
|
||||
/** Apple A11 processor (little cores). */
|
||||
cpuinfo_uarch_mistral = 0x00700106,
|
||||
/** Apple A12 processor (big cores). */
|
||||
cpuinfo_uarch_vortex = 0x00700107,
|
||||
/** Apple A12 processor (little cores). */
|
||||
cpuinfo_uarch_tempest = 0x00700108,
|
||||
/** Apple A13 processor (big cores). */
|
||||
cpuinfo_uarch_lightning = 0x00700109,
|
||||
/** Apple A13 processor (little cores). */
|
||||
cpuinfo_uarch_thunder = 0x0070010A,
|
||||
/** Apple M1 processor (big cores). */
|
||||
cpuinfo_uarch_firestorm = 0x0070010B,
|
||||
/** Apple M1 processor (little cores). */
|
||||
cpuinfo_uarch_icestorm = 0x0070010C,
|
||||
|
||||
/** Cavium ThunderX. */
|
||||
cpuinfo_uarch_thunderx = 0x00800100,
|
||||
/** Cavium ThunderX2 (originally Broadcom Vulkan). */
|
||||
cpuinfo_uarch_thunderx2 = 0x00800200,
|
||||
|
||||
/** Marvell PJ4. */
|
||||
cpuinfo_uarch_pj4 = 0x00900100,
|
||||
|
||||
/** Broadcom Brahma B15. */
|
||||
cpuinfo_uarch_brahma_b15 = 0x00A00100,
|
||||
/** Broadcom Brahma B53. */
|
||||
cpuinfo_uarch_brahma_b53 = 0x00A00101,
|
||||
|
||||
/** Applied Micro X-Gene. */
|
||||
cpuinfo_uarch_xgene = 0x00B00100,
|
||||
|
||||
/* Hygon Dhyana (a modification of AMD Zen for Chinese market). */
|
||||
cpuinfo_uarch_dhyana = 0x01000100,
|
||||
|
||||
/** HiSilicon TaiShan v110 (Huawei Kunpeng 920 series processors). */
|
||||
cpuinfo_uarch_taishan_v110 = 0x00C00100,
|
||||
};
|
||||
|
||||
#if defined(CPUIDINFO_ARCH_ARM)
|
||||
|
||||
#define CPUINFO_ARM_MIDR_IMPLEMENTER_MASK UINT32_C(0xFF000000)
|
||||
#define CPUINFO_ARM_MIDR_VARIANT_MASK UINT32_C(0x00F00000)
|
||||
#define CPUINFO_ARM_MIDR_ARCHITECTURE_MASK UINT32_C(0x000F0000)
|
||||
#define CPUINFO_ARM_MIDR_PART_MASK UINT32_C(0x0000FFF0)
|
||||
#define CPUINFO_ARM_MIDR_REVISION_MASK UINT32_C(0x0000000F)
|
||||
|
||||
#define CPUINFO_ARM_MIDR_IMPLEMENTER_OFFSET 24
|
||||
#define CPUINFO_ARM_MIDR_VARIANT_OFFSET 20
|
||||
#define CPUINFO_ARM_MIDR_ARCHITECTURE_OFFSET 16
|
||||
#define CPUINFO_ARM_MIDR_PART_OFFSET 4
|
||||
#define CPUINFO_ARM_MIDR_REVISION_OFFSET 0
|
||||
|
||||
inline static uint32_t midr_get_implementer(uint32_t midr) {
|
||||
return (midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) >> CPUINFO_ARM_MIDR_IMPLEMENTER_OFFSET;
|
||||
}
|
||||
|
||||
inline static uint32_t midr_get_part(uint32_t midr) {
|
||||
return (midr & CPUINFO_ARM_MIDR_PART_MASK) >> CPUINFO_ARM_MIDR_PART_OFFSET;
|
||||
}
|
||||
|
||||
inline static uint32_t midr_get_variant(uint32_t midr) {
|
||||
return (midr & CPUINFO_ARM_MIDR_VARIANT_MASK) >> CPUINFO_ARM_MIDR_VARIANT_OFFSET;
|
||||
}
|
||||
|
||||
static void decodeMIDR(
|
||||
uint32_t midr,
|
||||
uint32_t uarch[1]) {
|
||||
switch (midr_get_implementer(midr)) {
|
||||
case 'A':
|
||||
switch (midr_get_part(midr)) {
|
||||
//#if defined(_M_ARM) || defined(__arm__)
|
||||
case 0xC05:
|
||||
*uarch = cpuinfo_uarch_cortex_a5;
|
||||
break;
|
||||
case 0xC07:
|
||||
*uarch = cpuinfo_uarch_cortex_a7;
|
||||
break;
|
||||
case 0xC08:
|
||||
*uarch = cpuinfo_uarch_cortex_a8;
|
||||
break;
|
||||
case 0xC09:
|
||||
*uarch = cpuinfo_uarch_cortex_a9;
|
||||
break;
|
||||
case 0xC0C:
|
||||
*uarch = cpuinfo_uarch_cortex_a12;
|
||||
break;
|
||||
case 0xC0E:
|
||||
*uarch = cpuinfo_uarch_cortex_a17;
|
||||
break;
|
||||
case 0xC0D:
|
||||
/*
|
||||
* Rockchip RK3288 only.
|
||||
* Core information is ambiguous: some sources specify Cortex-A12, others - Cortex-A17.
|
||||
* Assume it is Cortex-A12.
|
||||
*/
|
||||
*uarch = cpuinfo_uarch_cortex_a12;
|
||||
break;
|
||||
case 0xC0F:
|
||||
*uarch = cpuinfo_uarch_cortex_a15;
|
||||
break;
|
||||
//#endif /* ARM */
|
||||
case 0xD01:
|
||||
*uarch = cpuinfo_uarch_cortex_a32;
|
||||
break;
|
||||
case 0xD03:
|
||||
*uarch = cpuinfo_uarch_cortex_a53;
|
||||
break;
|
||||
case 0xD04:
|
||||
*uarch = cpuinfo_uarch_cortex_a35;
|
||||
break;
|
||||
case 0xD05:
|
||||
// Note: use Variant, not Revision, field
|
||||
*uarch = (midr & CPUINFO_ARM_MIDR_VARIANT_MASK) == 0 ? cpuinfo_uarch_cortex_a55r0 : cpuinfo_uarch_cortex_a55;
|
||||
break;
|
||||
case 0xD06:
|
||||
*uarch = cpuinfo_uarch_cortex_a65;
|
||||
break;
|
||||
case 0xD07:
|
||||
*uarch = cpuinfo_uarch_cortex_a57;
|
||||
break;
|
||||
case 0xD08:
|
||||
*uarch = cpuinfo_uarch_cortex_a72;
|
||||
break;
|
||||
case 0xD09:
|
||||
*uarch = cpuinfo_uarch_cortex_a73;
|
||||
break;
|
||||
case 0xD0A:
|
||||
*uarch = cpuinfo_uarch_cortex_a75;
|
||||
break;
|
||||
case 0xD0B:
|
||||
*uarch = cpuinfo_uarch_cortex_a76;
|
||||
break;
|
||||
//#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__)
|
||||
case 0xD0C:
|
||||
*uarch = cpuinfo_uarch_neoverse_n1;
|
||||
break;
|
||||
//#endif /* ARM64 && !defined(__ANDROID__) */
|
||||
case 0xD0D:
|
||||
*uarch = cpuinfo_uarch_cortex_a77;
|
||||
break;
|
||||
case 0xD0E: /* Cortex-A76AE */
|
||||
*uarch = cpuinfo_uarch_cortex_a76;
|
||||
break;
|
||||
case 0xD41: /* Cortex-A78 */
|
||||
*uarch = cpuinfo_uarch_cortex_a78;
|
||||
break;
|
||||
case 0xD44: /* Cortex-X1 */
|
||||
*uarch = cpuinfo_uarch_cortex_x1;
|
||||
break;
|
||||
//#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__)
|
||||
case 0xD4A:
|
||||
*uarch = cpuinfo_uarch_neoverse_e1;
|
||||
break;
|
||||
//#endif /* ARM64 && !defined(__ANDROID__) */
|
||||
default:
|
||||
switch (midr_get_part(midr) >> 8) {
|
||||
//#if defined(_M_ARM) || defined(__arm__)
|
||||
case 7:
|
||||
*uarch = cpuinfo_uarch_arm7;
|
||||
break;
|
||||
case 9:
|
||||
*uarch = cpuinfo_uarch_arm9;
|
||||
break;
|
||||
case 11:
|
||||
*uarch = cpuinfo_uarch_arm11;
|
||||
break;
|
||||
//#endif /* ARM */
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown ARM CPU part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'B':
|
||||
switch (midr_get_part(midr)) {
|
||||
case 0x00F:
|
||||
*uarch = cpuinfo_uarch_brahma_b15;
|
||||
break;
|
||||
case 0x100:
|
||||
*uarch = cpuinfo_uarch_brahma_b53;
|
||||
break;
|
||||
//#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__)
|
||||
case 0x516:
|
||||
/* Broadcom Vulkan was sold to Cavium before it reached the market, so we identify it as Cavium ThunderX2 */
|
||||
*uarch = cpuinfo_uarch_thunderx2;
|
||||
break;
|
||||
//#endif
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown Broadcom CPU part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
break;
|
||||
//#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__)
|
||||
case 'C':
|
||||
switch (midr_get_part(midr)) {
|
||||
case 0x0A0: /* ThunderX */
|
||||
case 0x0A1: /* ThunderX 88XX */
|
||||
case 0x0A2: /* ThunderX 81XX */
|
||||
case 0x0A3: /* ThunderX 83XX */
|
||||
*uarch = cpuinfo_uarch_thunderx;
|
||||
break;
|
||||
case 0x0AF: /* ThunderX2 99XX */
|
||||
*uarch = cpuinfo_uarch_thunderx2;
|
||||
break;
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown Cavium CPU part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
break;
|
||||
//#endif
|
||||
case 'H':
|
||||
switch (midr_get_part(midr)) {
|
||||
//#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__)
|
||||
case 0xD01: /* Kunpeng 920 series */
|
||||
*uarch = cpuinfo_uarch_taishan_v110;
|
||||
break;
|
||||
//#endif
|
||||
case 0xD40: /* Kirin 980 Big/Medium cores -> Cortex-A76 */
|
||||
*uarch = cpuinfo_uarch_cortex_a76;
|
||||
break;
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown Huawei CPU part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
break;
|
||||
//#if defined(_M_ARM) || defined(__arm__)
|
||||
case 'i':
|
||||
switch (midr_get_part(midr) >> 8) {
|
||||
case 2: /* PXA 210/25X/26X */
|
||||
case 4: /* PXA 27X */
|
||||
case 6: /* PXA 3XX */
|
||||
*uarch = cpuinfo_uarch_xscale;
|
||||
break;
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown Intel CPU part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
break;
|
||||
//#endif /* ARM */
|
||||
case 'N':
|
||||
switch (midr_get_part(midr)) {
|
||||
case 0x000:
|
||||
*uarch = cpuinfo_uarch_denver;
|
||||
break;
|
||||
case 0x003:
|
||||
*uarch = cpuinfo_uarch_denver2;
|
||||
break;
|
||||
case 0x004:
|
||||
*uarch = cpuinfo_uarch_carmel;
|
||||
break;
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown Nvidia CPU part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
break;
|
||||
#if !defined(__ANDROID__)
|
||||
case 'P':
|
||||
switch (midr_get_part(midr)) {
|
||||
case 0x000:
|
||||
*uarch = cpuinfo_uarch_xgene;
|
||||
break;
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown Applied Micro CPU part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
break;
|
||||
#endif
|
||||
case 'Q':
|
||||
switch (midr_get_part(midr)) {
|
||||
// #if defined(_M_ARM) || defined(__arm__)
|
||||
case 0x00F:
|
||||
/* Mostly Scorpions, but some Cortex A5 may report this value as well */
|
||||
//if (has_vfpv4) {
|
||||
// /* Unlike Scorpion, Cortex-A5 comes with VFPv4 */
|
||||
// *vendor = cpuinfo_vendor_arm;
|
||||
// *uarch = cpuinfo_uarch_cortex_a5;
|
||||
//} else {
|
||||
*uarch = cpuinfo_uarch_scorpion;
|
||||
// }
|
||||
break;
|
||||
case 0x02D: /* Dual-core Scorpions */
|
||||
*uarch = cpuinfo_uarch_scorpion;
|
||||
break;
|
||||
case 0x04D:
|
||||
/*
|
||||
* Dual-core Krait:
|
||||
* - r1p0 -> Krait 200
|
||||
* - r1p4 -> Krait 200
|
||||
* - r2p0 -> Krait 300
|
||||
*/
|
||||
case 0x06F:
|
||||
/*
|
||||
* Quad-core Krait:
|
||||
* - r0p1 -> Krait 200
|
||||
* - r0p2 -> Krait 200
|
||||
* - r1p0 -> Krait 300
|
||||
* - r2p0 -> Krait 400 (Snapdragon 800 MSMxxxx)
|
||||
* - r2p1 -> Krait 400 (Snapdragon 801 MSMxxxxPRO)
|
||||
* - r3p1 -> Krait 450
|
||||
*/
|
||||
*uarch = cpuinfo_uarch_krait;
|
||||
break;
|
||||
//#endif /* ARM */
|
||||
case 0x201: /* Qualcomm Snapdragon 821: Low-power Kryo "Silver" */
|
||||
case 0x205: /* Qualcomm Snapdragon 820 & 821: High-performance Kryo "Gold" */
|
||||
case 0x211: /* Qualcomm Snapdragon 820: Low-power Kryo "Silver" */
|
||||
*uarch = cpuinfo_uarch_kryo;
|
||||
break;
|
||||
case 0x800: /* High-performance Kryo 260 (r10p2) / Kryo 280 (r10p1) "Gold" -> Cortex-A73 */
|
||||
*uarch = cpuinfo_uarch_cortex_a73;
|
||||
break;
|
||||
case 0x801: /* Low-power Kryo 260 / 280 "Silver" -> Cortex-A53 */
|
||||
*uarch = cpuinfo_uarch_cortex_a53;
|
||||
break;
|
||||
case 0x802: /* High-performance Kryo 385 "Gold" -> Cortex-A75 */
|
||||
*uarch = cpuinfo_uarch_cortex_a75;
|
||||
break;
|
||||
case 0x803: /* Low-power Kryo 385 "Silver" -> Cortex-A55r0 */
|
||||
*uarch = cpuinfo_uarch_cortex_a55r0;
|
||||
break;
|
||||
case 0x804: /* High-performance Kryo 485 "Gold" / "Gold Prime" -> Cortex-A76 */
|
||||
*uarch = cpuinfo_uarch_cortex_a76;
|
||||
break;
|
||||
case 0x805: /* Low-performance Kryo 485 "Silver" -> Cortex-A55 */
|
||||
*uarch = cpuinfo_uarch_cortex_a55;
|
||||
break;
|
||||
//#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__)
|
||||
case 0xC00:
|
||||
*uarch = cpuinfo_uarch_falkor;
|
||||
break;
|
||||
case 0xC01:
|
||||
*uarch = cpuinfo_uarch_saphira;
|
||||
break;
|
||||
//#endif /* ARM64 && !defined(__ANDROID__) */
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown Qualcomm CPU part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
break;
|
||||
case 'S':
|
||||
switch (midr & (CPUINFO_ARM_MIDR_VARIANT_MASK | CPUINFO_ARM_MIDR_PART_MASK)) {
|
||||
case 0x00100010:
|
||||
/*
|
||||
* Exynos 8890 MIDR = 0x531F0011, assume Exynos M1 has:
|
||||
* - CPU variant 0x1
|
||||
* - CPU part 0x001
|
||||
*/
|
||||
*uarch = cpuinfo_uarch_exynos_m1;
|
||||
break;
|
||||
case 0x00400010:
|
||||
/*
|
||||
* Exynos 8895 MIDR = 0x534F0010, assume Exynos M2 has:
|
||||
* - CPU variant 0x4
|
||||
* - CPU part 0x001
|
||||
*/
|
||||
*uarch = cpuinfo_uarch_exynos_m2;
|
||||
break;
|
||||
case 0x00100020:
|
||||
/*
|
||||
* Exynos 9810 MIDR = 0x531F0020, assume Exynos M3 has:
|
||||
* - CPU variant 0x1
|
||||
* - CPU part 0x002
|
||||
*/
|
||||
*uarch = cpuinfo_uarch_exynos_m3;
|
||||
break;
|
||||
case 0x00100030:
|
||||
/*
|
||||
* Exynos 9820 MIDR = 0x531F0030, assume Exynos M4 has:
|
||||
* - CPU variant 0x1
|
||||
* - CPU part 0x003
|
||||
*/
|
||||
*uarch = cpuinfo_uarch_exynos_m4;
|
||||
break;
|
||||
case 0x00100040:
|
||||
/*
|
||||
* Exynos 9820 MIDR = 0x531F0040, assume Exynos M5 has:
|
||||
* - CPU variant 0x1
|
||||
* - CPU part 0x004
|
||||
*/
|
||||
*uarch = cpuinfo_uarch_exynos_m5;
|
||||
break;
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown Samsung CPU variant 0x"
|
||||
<< std::hex << midr_get_variant(midr) << " part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
break;
|
||||
//#if defined(_M_ARM) || defined(__arm__)
|
||||
case 'V':
|
||||
switch (midr_get_part(midr)) {
|
||||
case 0x581: /* PJ4 / PJ4B */
|
||||
case 0x584: /* PJ4B-MP / PJ4C */
|
||||
*uarch = cpuinfo_uarch_pj4;
|
||||
break;
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown Marvell CPU part 0x" << std::hex << midr_get_part(midr) << " ignored";
|
||||
}
|
||||
break;
|
||||
//#endif /* ARM */
|
||||
default:
|
||||
LOGS_DEFAULT(WARNING) << "unknown CPU uarch from MIDR value: 0x" << std::hex << midr;
|
||||
}
|
||||
}
|
||||
|
||||
#endif // arm or arm64
|
||||
|
|
@ -482,7 +482,7 @@ MlasConvSym(
|
|||
// s8s8 under ARM64
|
||||
#if defined(MLAS_TARGET_ARM64)
|
||||
const auto Kernel =
|
||||
(Params.InputIsSigned && (GetMlasPlatform().GetCoreType() == mlas_core_little))
|
||||
(Params.InputIsSigned && MLAS_CPUIDINFO::GetCPUIDInfo().IsCurrentCoreArmv8NarrowLd())
|
||||
? ConvSymDispatch->KernelLittle
|
||||
: ConvSymDispatch->Kernel;
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -104,22 +104,11 @@ Abstract:
|
|||
#if !defined(BUILD_MLAS_NO_ONNXRUNTIME)
|
||||
#include "core/platform/threadpool.h"
|
||||
|
||||
#if defined(MLAS_TARGET_ARM64) && defined(__linux__)
|
||||
|
||||
#include "core/common/cpuid_info.h"
|
||||
using MLAS_CPUIDINFO = onnxruntime::CPUIDInfo;
|
||||
|
||||
#include <unistd.h>
|
||||
#include <sys/syscall.h>
|
||||
#if !defined(__NR_getcpu)
|
||||
#include <asm-generic/unistd.h>
|
||||
#endif
|
||||
|
||||
#endif // MLAS_TARGET_ARM64
|
||||
|
||||
#else // BUILD_MLAS_NO_ONNXRUNTIME
|
||||
|
||||
#if defined(MLAS_TARGET_ARM64) && defined(__linux__)
|
||||
class MLASCPUIDInfo
|
||||
{
|
||||
public:
|
||||
|
|
@ -132,7 +121,15 @@ class MLASCPUIDInfo
|
|||
// ARM
|
||||
bool HasArmNeonDot() const { return has_arm_neon_dot_; }
|
||||
|
||||
int32_t GetCurrentUarch() { return -1; }
|
||||
uint32_t GetCurrentCoreIdx() const { return 0xFFFFFFFF; }
|
||||
|
||||
int32_t GetCurrentUarch() const { return -1; }
|
||||
|
||||
int32_t GetCoreUarch(uint32_t coreId) const { return -1; }
|
||||
|
||||
bool IsCoreArmv8NarrowLd(uint32_t coreId) const { return false; }
|
||||
|
||||
bool IsCurrentCoreArmv8NarrowLd() const { return false; }
|
||||
|
||||
private:
|
||||
MLASCPUIDInfo();
|
||||
|
|
@ -141,6 +138,45 @@ class MLASCPUIDInfo
|
|||
};
|
||||
using MLAS_CPUIDINFO = MLASCPUIDInfo;
|
||||
|
||||
#if defined(MLAS_TARGET_ARM64)
|
||||
/**
|
||||
* @brief IDs for cpu microarchitectures.
|
||||
*
|
||||
* Copied from python cpuinfo package. Can't use the definition
|
||||
* from cpuinfo directly as it causes lots of compilation issues
|
||||
* in many platforms that we support.
|
||||
*/
|
||||
enum MlasUArch {
|
||||
cpuinfo_uarch_unknown = 0,
|
||||
|
||||
/** ARM Cortex-A32. */
|
||||
cpuinfo_uarch_cortex_a32 = 0x00300332,
|
||||
/** ARM Cortex-A35. */
|
||||
cpuinfo_uarch_cortex_a35 = 0x00300335,
|
||||
/** ARM Cortex-A53. */
|
||||
cpuinfo_uarch_cortex_a53 = 0x00300353,
|
||||
/** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities compared to revision 1+). */
|
||||
cpuinfo_uarch_cortex_a55r0 = 0x00300354,
|
||||
/** ARM Cortex-A55. */
|
||||
cpuinfo_uarch_cortex_a55 = 0x00300355,
|
||||
/** ARM Cortex-A57. */
|
||||
cpuinfo_uarch_cortex_a57 = 0x00300357,
|
||||
/** ARM Cortex-A65. */
|
||||
cpuinfo_uarch_cortex_a65 = 0x00300365,
|
||||
/** ARM Cortex-A72. */
|
||||
cpuinfo_uarch_cortex_a72 = 0x00300372,
|
||||
/** ARM Cortex-A73. */
|
||||
cpuinfo_uarch_cortex_a73 = 0x00300373,
|
||||
/** ARM Cortex-A75. */
|
||||
cpuinfo_uarch_cortex_a75 = 0x00300375,
|
||||
/** ARM Cortex-A76. */
|
||||
cpuinfo_uarch_cortex_a76 = 0x00300376,
|
||||
/** ARM Cortex-A77. */
|
||||
cpuinfo_uarch_cortex_a77 = 0x00300377,
|
||||
/** ARM Cortex-A78. */
|
||||
cpuinfo_uarch_cortex_a78 = 0x00300378,
|
||||
};
|
||||
|
||||
#endif // MLAS_TARGET_ARM64
|
||||
|
||||
#endif // BUILD_MLAS_NO_ONNXRUNTIME
|
||||
|
|
@ -785,44 +821,6 @@ struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
|
|||
// Environment information class.
|
||||
//
|
||||
|
||||
/**
|
||||
* @brief IDs for cpu microarchitectures.
|
||||
*
|
||||
* Copied from python cpuinfo package. Can't use the definition
|
||||
* from cpuinfo directly as it causes lots of compilation issues
|
||||
* in many platforms that we support.
|
||||
*/
|
||||
enum MlasUArch {
|
||||
cpuinfo_uarch_unknown = 0,
|
||||
|
||||
/** ARM Cortex-A32. */
|
||||
cpuinfo_uarch_cortex_a32 = 0x00300332,
|
||||
/** ARM Cortex-A35. */
|
||||
cpuinfo_uarch_cortex_a35 = 0x00300335,
|
||||
/** ARM Cortex-A53. */
|
||||
cpuinfo_uarch_cortex_a53 = 0x00300353,
|
||||
/** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities compared to revision 1+). */
|
||||
cpuinfo_uarch_cortex_a55r0 = 0x00300354,
|
||||
/** ARM Cortex-A55. */
|
||||
cpuinfo_uarch_cortex_a55 = 0x00300355,
|
||||
/** ARM Cortex-A57. */
|
||||
cpuinfo_uarch_cortex_a57 = 0x00300357,
|
||||
/** ARM Cortex-A65. */
|
||||
cpuinfo_uarch_cortex_a65 = 0x00300365,
|
||||
/** ARM Cortex-A72. */
|
||||
cpuinfo_uarch_cortex_a72 = 0x00300372,
|
||||
/** ARM Cortex-A73. */
|
||||
cpuinfo_uarch_cortex_a73 = 0x00300373,
|
||||
/** ARM Cortex-A75. */
|
||||
cpuinfo_uarch_cortex_a75 = 0x00300375,
|
||||
/** ARM Cortex-A76. */
|
||||
cpuinfo_uarch_cortex_a76 = 0x00300376,
|
||||
/** ARM Cortex-A77. */
|
||||
cpuinfo_uarch_cortex_a77 = 0x00300377,
|
||||
/** ARM Cortex-A78. */
|
||||
cpuinfo_uarch_cortex_a78 = 0x00300378,
|
||||
};
|
||||
|
||||
enum MlasCoreType { mlas_core_unknown = 0, mlas_core_little = 2, mlas_core_big = 3 };
|
||||
|
||||
|
||||
|
|
@ -891,51 +889,6 @@ struct MLAS_PLATFORM {
|
|||
static constexpr int32_t MaximumThreadCount = MLAS_MAXIMUM_THREAD_COUNT;
|
||||
#endif
|
||||
|
||||
#if defined(MLAS_TARGET_ARM64) && defined(__linux__)
|
||||
// TODO!! implement uarch detection in Windows
|
||||
std::vector<MlasCoreType> mlas_coretype_tbl;
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @return 2 current core is little core with narrow memory load (e.g. ARMv8 a53)
|
||||
* 3 current core is big core with wider load (e.g. ARMv8 a72)
|
||||
*/
|
||||
MlasCoreType GetCoreType()
|
||||
{
|
||||
#if defined(MLAS_TARGET_ARM64) && defined(__linux__)
|
||||
|
||||
if (mlas_coretype_tbl.size() == 0) {
|
||||
// functionality missing, return default
|
||||
return mlas_core_big;
|
||||
}
|
||||
|
||||
unsigned cpu = 0;
|
||||
if (syscall(__NR_getcpu, &cpu, NULL, NULL) != 0) {
|
||||
// failed to detect current core id. give up
|
||||
return mlas_core_big;
|
||||
}
|
||||
|
||||
if (cpu >= mlas_coretype_tbl.size()) {
|
||||
mlas_coretype_tbl.resize(cpu + 1, mlas_core_unknown);
|
||||
}
|
||||
|
||||
auto core_type = mlas_coretype_tbl[cpu];
|
||||
if (core_type == mlas_core_unknown) {
|
||||
auto uarch = MLAS_CPUIDINFO::GetCPUIDInfo().GetCurrentUarch();
|
||||
if (uarch == cpuinfo_uarch_cortex_a53 || uarch == cpuinfo_uarch_cortex_a55r0 ||
|
||||
uarch == cpuinfo_uarch_cortex_a55) {
|
||||
core_type = mlas_core_little;
|
||||
} else {
|
||||
core_type = mlas_core_big;
|
||||
}
|
||||
mlas_coretype_tbl[cpu] = core_type;
|
||||
}
|
||||
return core_type;
|
||||
|
||||
#else
|
||||
return mlas_core_big;
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
inline
|
||||
|
|
|
|||
|
|
@ -26,11 +26,21 @@ Abstract:
|
|||
|
||||
#if defined(MLAS_TARGET_ARM64)
|
||||
#if defined(_WIN32)
|
||||
|
||||
// N.B. Support building with downlevel versions of the Windows SDK.
|
||||
#ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE
|
||||
#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
|
||||
#endif
|
||||
|
||||
#if defined(BUILD_MLAS_NO_ONNXRUNTIME)
|
||||
MLASCPUIDInfo::MLASCPUIDInfo()
|
||||
{
|
||||
has_arm_neon_dot_ = (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
#elif defined(__linux__)
|
||||
|
||||
#include <sys/auxv.h>
|
||||
#include <asm/hwcap.h>
|
||||
// N.B. Support building with older versions of asm/hwcap.h that do not define
|
||||
|
|
@ -43,7 +53,19 @@ Abstract:
|
|||
MLASCPUIDInfo::MLASCPUIDInfo() { has_arm_neon_dot_ = ((getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0); }
|
||||
#endif
|
||||
|
||||
#else
|
||||
|
||||
#if defined(BUILD_MLAS_NO_ONNXRUNTIME)
|
||||
MLASCPUIDInfo::MLASCPUIDInfo() {}
|
||||
#endif
|
||||
|
||||
#endif // Windows vs Linux vs Unknown
|
||||
#else // not MLAS_TARGET_ARM64
|
||||
|
||||
#if defined(BUILD_MLAS_NO_ONNXRUNTIME)
|
||||
MLASCPUIDInfo::MLASCPUIDInfo() {}
|
||||
#endif
|
||||
|
||||
#endif // MLAS_TARGET_ARM64
|
||||
|
||||
#ifdef MLAS_TARGET_AMD64_IX86
|
||||
|
|
@ -411,14 +433,6 @@ Return Value:
|
|||
#endif // __linux__
|
||||
#endif // MLAS_TARGET_POWER
|
||||
|
||||
// Init the table describing the type (big or litte) of each core
|
||||
#if defined(MLAS_TARGET_ARM64) && defined(__linux__)
|
||||
// TODO!! implemente core uarch detection in Windows
|
||||
auto tbl_size = std::thread::hardware_concurrency();
|
||||
if (tbl_size > 0) {
|
||||
mlas_coretype_tbl.resize(tbl_size, mlas_core_unknown);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t
|
||||
|
|
|
|||
|
|
@ -209,9 +209,9 @@ MlasSymmQgemmBatch(
|
|||
if (ThreadPool == nullptr) {
|
||||
// So our caller handles threaded job partition.
|
||||
// Call single threaded operation directly
|
||||
auto uarch = GetMlasPlatform().GetCoreType();
|
||||
auto uarch = MLAS_CPUIDINFO::GetCPUIDInfo().IsCurrentCoreArmv8NarrowLd();
|
||||
MLAS_SYMM_QGEMM_OPERATION* operation =
|
||||
uarch == mlas_core_little ? dispatch->LitOperation : dispatch->BigOperation;
|
||||
uarch ? dispatch->LitOperation : dispatch->BigOperation;
|
||||
|
||||
for (size_t gemm_i = 0; gemm_i < BatchN; gemm_i++) {
|
||||
auto Data = &DataParams[gemm_i];
|
||||
|
|
@ -260,9 +260,9 @@ MlasSymmQgemmBatch(
|
|||
ThreadsPerGemm = ThreadCountM * ThreadCountN;
|
||||
|
||||
MlasTrySimpleParallel(ThreadPool, ThreadsPerGemm * BatchN, [&](ptrdiff_t tid) {
|
||||
auto uarch = GetMlasPlatform().GetCoreType();
|
||||
auto uarch = MLAS_CPUIDINFO::GetCPUIDInfo().IsCurrentCoreArmv8NarrowLd();
|
||||
MLAS_SYMM_QGEMM_OPERATION* operation =
|
||||
uarch == mlas_core_little ? dispatch->LitOperation : dispatch->BigOperation;
|
||||
uarch ? dispatch->LitOperation : dispatch->BigOperation;
|
||||
|
||||
const auto gemm_i = tid / ThreadsPerGemm;
|
||||
const auto blk_i = tid % ThreadsPerGemm;
|
||||
|
|
|
|||
Loading…
Reference in a new issue