diff --git a/onnxruntime/core/common/cpuid_info.cc b/onnxruntime/core/common/cpuid_info.cc index f94720f05b..7327942177 100644 --- a/onnxruntime/core/common/cpuid_info.cc +++ b/onnxruntime/core/common/cpuid_info.cc @@ -1,20 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - #include "core/common/cpuid_info.h" #include "core/common/logging/logging.h" #include "core/common/logging/severity.h" -#if defined(CPUIDINFO_ARCH_X86) -#include -#if defined(_MSC_VER) -#include -#elif defined(__GNUC__) -#include -#endif +#ifdef __linux__ + +#include +#include +#if !defined(__NR_getcpu) +#include #endif -#if defined(CPUIDINFO_ARCH_ARM) && defined(CPUINFO_SUPPORTED) && defined(__linux__) +#if defined(CPUIDINFO_ARCH_ARM) + #include #include // N.B. Support building with older versions of asm/hwcap.h that do not define @@ -23,20 +22,40 @@ #define HWCAP_ASIMDDP (1 << 20) #endif -#endif +#endif // ARM -#include +#endif // Linux #if _WIN32 + +#include "Windows.h" + #define HAS_WINDOWS_DESKTOP WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + +#ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE +#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43 #endif -#if (defined(CPUIDINFO_ARCH_X86) || defined(CPUIDINFO_ARCH_ARM)) && defined(CPUINFO_SUPPORTED) && (!_WIN32 || defined(HAS_WINDOWS_DESKTOP)) + +#endif // _WIN32 + +#if defined(CPUINFO_SUPPORTED) #include -#endif +#else +#include "core/common/cpuid_uarch.h" +#endif // CPUINFO_SUPPORTED + namespace onnxruntime { -#if defined(CPUIDINFO_ARCH_X86) +#ifdef CPUIDINFO_ARCH_X86 + +#include +#if defined(_MSC_VER) +#include +#elif defined(__GNUC__) +#include +#endif + static inline void GetCPUID(int function_id, int data[4]) { // NOLINT #if defined(_MSC_VER) __cpuid(reinterpret_cast(data), function_id); @@ -56,83 +75,158 @@ static inline int XGETBV() { return eax; #endif } -#endif // CPUIDINFO_ARCH_X86 +void CPUIDInfo::X86Init() { + int data[4] = {-1}; + GetCPUID(0, data); -CPUIDInfo::CPUIDInfo() { -#if (defined(CPUIDINFO_ARCH_X86) || defined(CPUIDINFO_ARCH_ARM)) && defined(CPUINFO_SUPPORTED) + int num_IDs = data[0]; + if (num_IDs >= 1) { + GetCPUID(1, data); + if (data[2] & (1 << 27)) { + constexpr int AVX_MASK = 0x6; + constexpr int AVX512_MASK = 0xE6; + int value = XGETBV(); + bool has_sse2 = (data[3] & (1 << 26)); + has_sse3_ = (data[2] & 0x1); + has_sse4_1_ = (data[2] & (1 << 19)); + bool has_ssse3 = (data[2] & (1 << 9)); + has_avx_ = has_sse2 && has_ssse3 && (data[2] & (1 << 28)) && ((value & AVX_MASK) == AVX_MASK); + bool has_avx512 = (value & AVX512_MASK) == AVX512_MASK; + has_f16c_ = has_avx_ && (data[2] & (1 << 29)) && (data[3] & (1 << 26)); + + if (num_IDs >= 7) { + GetCPUID(7, data); + has_avx2_ = has_avx_ && (data[1] & (1 << 5)); + has_avx512f_ = has_avx512 && (data[1] & (1 << 16)); + // Add check for AVX512 Skylake since tensorization GEMM need intrinsics from avx512bw/avx512dq. + // avx512_skylake = avx512f | avx512vl | avx512cd | avx512bw | avx512dq + has_avx512_skylake_ = has_avx512 && (data[1] & ((1 << 16) | (1 << 17) | (1 << 28) | (1 << 30) | (1 << 31))); + is_hybrid_ = (data[3] & (1 << 15)); + } + } + } +} + +#endif /* CPUIDINFO_ARCH_X86 */ + +#if defined(CPUIDINFO_ARCH_ARM) +#ifdef __linux__ + +void CPUIDInfo::ArmLinuxInit() { + // Pytorch CPUINFO only works on ARM linux or android + // Assuming no hyper-threading, no NUMA groups +#ifdef CPUINFO_SUPPORTED pytorch_cpuinfo_init_ = cpuinfo_initialize(); if (!pytorch_cpuinfo_init_) { LOGS_DEFAULT(WARNING) << "Failed to init pytorch cpuinfo library, may cause CPU EP performance degradation due to undetected CPU features."; + return; } +#else + pytorch_cpuinfo_init_ = false; #endif -#if defined(CPUIDINFO_ARCH_X86) - int data[4] = {-1}; - GetCPUID(0, data); - - int num_IDs = data[0]; - if (num_IDs >= 1) { - GetCPUID(1, data); - if (data[2] & (1 << 27)) { - constexpr int AVX_MASK = 0x6; - constexpr int AVX512_MASK = 0xE6; - int value = XGETBV(); - bool has_sse2 = (data[3] & (1 << 26)); - has_sse3_ = (data[2] & 0x1); - has_sse4_1_ = (data[2] & (1 << 19)); - bool has_ssse3 = (data[2] & (1 << 9)); - has_avx_ = has_sse2 && has_ssse3 && (data[2] & (1 << 28)) && ((value & AVX_MASK) == AVX_MASK); - bool has_avx512 = (value & AVX512_MASK) == AVX512_MASK; - has_f16c_ = has_avx_ && (data[2] & (1 << 29)) && (data[3] & (1 << 26)); - - if (num_IDs >= 7) { - GetCPUID(7, data); - has_avx2_ = has_avx_ && (data[1] & (1 << 5)); - has_avx512f_ = has_avx512 && (data[1] & (1 << 16)); - // Add check for AVX512 Skylake since tensorization GEMM need intrinsics from avx512bw/avx512dq. - // avx512_skylake = avx512f | avx512vl | avx512cd | avx512bw | avx512dq - has_avx512_skylake_ = has_avx512 && (data[1] & ((1 << 16) | (1 << 17) | (1 << 28) | (1 << 30) | (1 << 31))); - is_hybrid_ = (data[3] & (1 << 15)); - } + if (pytorch_cpuinfo_init_) { + is_hybrid_ = cpuinfo_get_uarchs_count() > 1; + has_arm_neon_dot_ = cpuinfo_has_arm_neon_dot(); + const uint32_t core_cnt = cpuinfo_get_cores_count(); + core_uarchs_.resize(core_cnt, cpuinfo_uarch_unknown); + is_armv8_narrow_ld_.resize(core_cnt, false); + for (uint32_t c = 0; c < core_cnt; c++) { + const struct cpuinfo_processor* proc = cpuinfo_get_processor(c); + if (proc == nullptr) { + continue; + } + const struct cpuinfo_core* corep = proc->core; + if (corep == nullptr) { + continue; + } + auto coreid = proc->linux_id; + auto uarch = corep->uarch; + core_uarchs_[coreid] = uarch; + if (uarch == cpuinfo_uarch_cortex_a53 || uarch == cpuinfo_uarch_cortex_a55r0 || + uarch == cpuinfo_uarch_cortex_a55) { + is_armv8_narrow_ld_[coreid] = true; } } -#endif - -#if defined(CPUIDINFO_ARCH_ARM) -#ifdef HWCAP_ASIMDDP - - // only works on ARM linux or android, does not work on Windows - if (pytorch_cpuinfo_init_) { - is_hybrid_ = cpuinfo_get_uarchs_count() > 1; - has_arm_neon_dot_ = cpuinfo_has_arm_neon_dot(); - } else { - has_arm_neon_dot_ = ((getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0); - } + } else { + has_arm_neon_dot_ = ((getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0); + } +} #elif defined(_WIN32) - // TODO implement hardware feature detection in windows. - is_hybrid_ = true; -#endif -#endif +void CPUIDInfo::ArmWindowsInit() { + +#pragma region Application Family or OneCore Family +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM) + // Read MIDR from windows registry + // TODO!! Don't support multiple processor group yet!! + constexpr int MAX_CORES = 64; + constexpr int MAX_VALUE_NAME = 4096; + + TCHAR midrKey[MAX_VALUE_NAME] = ""; // buffer for processor registry name + uint32_t lastUarch = cpuinfo_uarch_unknown; + for (int i = 0; i < MAX_CORES - 1; i++) { + snprintf(midrKey, MAX_VALUE_NAME, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\%d", i); + uint64_t midrVal; + unsigned long midrSize = sizeof(uint64_t); + + /* + * ARM lists for each coprocessor register 5 fields: op0/op1/CRn/CRm/op2. + * You need to put those numbers through the ARM64_SYSREG macro: + * + * #define ARM64_SYSREG(op0, op1, crn, crm, op2) \ + * (((op0 & 1) << 14) | \ + * ((op1 & 7) << 11) | \ + * ((crn & 15) << 7) | \ + * ((crm & 15) << 3) | \ + * ((op2 & 7) << 0)) + * + * For the CP value of MIDR, op0 = 3 and the others are all = 0, so we come up with 0x4000, + */ + auto retCode = ::RegGetValue(HKEY_LOCAL_MACHINE, midrKey, "CP 4000", RRF_RT_REG_QWORD, nullptr, &midrVal, &midrSize); + if (retCode != ERROR_SUCCESS) { + break; + } + uint32_t uarch = cpuinfo_uarch_unknown; + decodeMIDR((uint32_t)midrVal, &uarch); + core_uarchs_.push_back(uarch); + if (uarch == cpuinfo_uarch_cortex_a53 || uarch == cpuinfo_uarch_cortex_a55r0 || + uarch == cpuinfo_uarch_cortex_a55) { + is_armv8_narrow_ld_.push_back(true); + } else { + is_armv8_narrow_ld_.push_back(false); + } + + if (i == 0) { + lastUarch = uarch; + } else if (lastUarch != uarch) { + is_hybrid_ = true; + lastUarch = uarch; + } + } +#endif /* Application Family or OneCore Family */ + + has_arm_neon_dot_ = (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0); } -int32_t CPUIDInfo::GetCurrentUarch() const { -#if (defined(CPUIDINFO_ARCH_X86) || defined(CPUIDINFO_ARCH_ARM)) && defined(CPUINFO_SUPPORTED) - if (!pytorch_cpuinfo_init_) { - return -1; - } - const auto uarchIdx = cpuinfo_get_current_uarch_index(); - const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(uarchIdx); - if (uarch_info == NULL) { - return -1; - } - return uarch_info->uarch; +#endif /* (arm or arm64) and windows */ +#endif /* arm or arm64*/ +uint32_t CPUIDInfo::GetCurrentCoreIdx() const { +#ifdef _WIN32 + return GetCurrentProcessorNumber(); +#elif defined(__linux__) + uint32_t coreIdx = 0xFFFFFFFF; + if (syscall(__NR_getcpu, &coreIdx, NULL, NULL) != 0) { + // failed to detect current core id. give up + return 0xFFFFFFFF; + } + return coreIdx; #else - return -1; + return 0xFFFFFFFF; // don't know how to get core index #endif } -} // namespace onnxruntime +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/common/cpuid_info.h b/onnxruntime/core/common/cpuid_info.h index aa0cc485e9..f76f0b0a15 100644 --- a/onnxruntime/core/common/cpuid_info.h +++ b/onnxruntime/core/common/cpuid_info.h @@ -11,7 +11,7 @@ #if defined(_M_ARM64) || defined(__aarch64__) || defined(_M_ARM) || defined(__arm__) #define CPUIDINFO_ARCH_ARM -#endif +#endif // ARM or ARM64 namespace onnxruntime { @@ -34,13 +34,76 @@ class CPUIDInfo { // ARM bool HasArmNeonDot() const { return has_arm_neon_dot_; } + uint32_t GetCurrentCoreIdx() const; + /** * @return CPU core micro-architecture running the current thread */ - int32_t GetCurrentUarch() const; + int32_t GetCurrentUarch() const { + if (core_uarchs_.empty()) { + return -1; + } + + uint32_t coreIdx = GetCurrentCoreIdx(); + if (coreIdx >= core_uarchs_.size()) { + return -1; + } + return core_uarchs_[coreIdx]; + } + + /** + * @return CPU core micro-architecture + */ + int32_t GetCoreUarch(uint32_t coreId) const { + if (coreId >= core_uarchs_.size()) { + return -1; + } + return core_uarchs_[coreId]; + } + + /** + * @brief Some ARMv8 power efficient core has narrower 64b load/store + * that needs specialized optimiztion in kernels + * @return whether the indicated core has narrower load/store device + */ + bool IsCoreArmv8NarrowLd(uint32_t coreId) const { + if (coreId >= is_armv8_narrow_ld_.size()) { + return false; + } + return is_armv8_narrow_ld_[coreId]; + } + + /** + * @brief Some ARMv8 power efficient core has narrower 64b load/store + * that needs specialized optimiztion in kernels + * @return whether the current core has narrower load/store device + */ + bool IsCurrentCoreArmv8NarrowLd() const { + if (is_armv8_narrow_ld_.empty()) { + return false; + } + + uint32_t coreIdx = GetCurrentCoreIdx(); + if (coreIdx >= is_armv8_narrow_ld_.size()) { + return false; + } + return is_armv8_narrow_ld_[coreIdx]; + } + private: - CPUIDInfo(); + CPUIDInfo() { +#ifdef CPUIDINFO_ARCH_X86 + X86Init(); +#elif defined(CPUIDINFO_ARCH_ARM) +#ifdef __linux__ + ArmLinuxInit(); +#elif defined(_WIN32) + ArmWindowsInit(); +#endif /* (arm or arm64) and windows */ +#endif + + } bool has_avx_{false}; bool has_avx2_{false}; bool has_avx512f_{false}; @@ -50,10 +113,32 @@ class CPUIDInfo { bool has_sse4_1_{false}; bool is_hybrid_{false}; -#if (defined(CPUIDINFO_ARCH_X86) || defined(CPUIDINFO_ARCH_ARM)) && defined(CPUINFO_SUPPORTED) - bool pytorch_cpuinfo_init_{false}; -#endif + std::vector core_uarchs_; // micro-arch of each core + + // In ARMv8 systems, some power efficient cores has narrower + // 64b load/store devices. It takes longer for them to load + // 128b vectore registers. + std::vector is_armv8_narrow_ld_; + bool has_arm_neon_dot_{false}; + +#ifdef CPUIDINFO_ARCH_X86 + + void X86Init(); + +#elif defined(CPUIDINFO_ARCH_ARM) +#ifdef __linux__ + + bool pytorch_cpuinfo_init_{false}; + void ArmLinuxInit(); + +#elif defined(_WIN32) + + void ArmWindowsInit(); + +#endif /* (arm or arm64) and windows */ +#endif + }; } // namespace onnxruntime diff --git a/onnxruntime/core/common/cpuid_uarch.h b/onnxruntime/core/common/cpuid_uarch.h new file mode 100644 index 0000000000..73a05c4b91 --- /dev/null +++ b/onnxruntime/core/common/cpuid_uarch.h @@ -0,0 +1,533 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + cpuid_uarch.h + +Abstract: + ARM Processor microarchitecture descriptions and utilities + + Processors with different microarchitectures often have different + instruction performance characteristics, and may have dramatically + different pipeline organization. + + We leverage Pytorch CPUinfo (https://github.com/pytorch/cpuinfo) for + hardware feature detection. Unfortunately CPUinfo package only works + on a few platforms but we need to support more. This ugly hack is + here to bridge the gap. + +--*/ + +enum CPUIDINFOuarch { + /** Microarchitecture is unknown, or the library failed to get information about the microarchitecture from OS */ + cpuinfo_uarch_unknown = 0, + + /** Intel/Marvell XScale series. */ + cpuinfo_uarch_xscale = 0x00100600, + + /** ARM7 series. */ + cpuinfo_uarch_arm7 = 0x00300100, + /** ARM9 series. */ + cpuinfo_uarch_arm9 = 0x00300101, + /** ARM 1136, ARM 1156, ARM 1176, or ARM 11MPCore. */ + cpuinfo_uarch_arm11 = 0x00300102, + + /** ARM Cortex-A5. */ + cpuinfo_uarch_cortex_a5 = 0x00300205, + /** ARM Cortex-A7. */ + cpuinfo_uarch_cortex_a7 = 0x00300207, + /** ARM Cortex-A8. */ + cpuinfo_uarch_cortex_a8 = 0x00300208, + /** ARM Cortex-A9. */ + cpuinfo_uarch_cortex_a9 = 0x00300209, + /** ARM Cortex-A12. */ + cpuinfo_uarch_cortex_a12 = 0x00300212, + /** ARM Cortex-A15. */ + cpuinfo_uarch_cortex_a15 = 0x00300215, + /** ARM Cortex-A17. */ + cpuinfo_uarch_cortex_a17 = 0x00300217, + + /** ARM Cortex-A32. */ + cpuinfo_uarch_cortex_a32 = 0x00300332, + /** ARM Cortex-A35. */ + cpuinfo_uarch_cortex_a35 = 0x00300335, + /** ARM Cortex-A53. */ + cpuinfo_uarch_cortex_a53 = 0x00300353, + /** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities compared to revision 1+). */ + cpuinfo_uarch_cortex_a55r0 = 0x00300354, + /** ARM Cortex-A55. */ + cpuinfo_uarch_cortex_a55 = 0x00300355, + /** ARM Cortex-A57. */ + cpuinfo_uarch_cortex_a57 = 0x00300357, + /** ARM Cortex-A65. */ + cpuinfo_uarch_cortex_a65 = 0x00300365, + /** ARM Cortex-A72. */ + cpuinfo_uarch_cortex_a72 = 0x00300372, + /** ARM Cortex-A73. */ + cpuinfo_uarch_cortex_a73 = 0x00300373, + /** ARM Cortex-A75. */ + cpuinfo_uarch_cortex_a75 = 0x00300375, + /** ARM Cortex-A76. */ + cpuinfo_uarch_cortex_a76 = 0x00300376, + /** ARM Cortex-A77. */ + cpuinfo_uarch_cortex_a77 = 0x00300377, + /** ARM Cortex-A78. */ + cpuinfo_uarch_cortex_a78 = 0x00300378, + + /** ARM Neoverse N1. */ + cpuinfo_uarch_neoverse_n1 = 0x00300400, + /** ARM Neoverse E1. */ + cpuinfo_uarch_neoverse_e1 = 0x00300401, + + /** ARM Cortex-X1. */ + cpuinfo_uarch_cortex_x1 = 0x00300500, + + /** Qualcomm Scorpion. */ + cpuinfo_uarch_scorpion = 0x00400100, + /** Qualcomm Krait. */ + cpuinfo_uarch_krait = 0x00400101, + /** Qualcomm Kryo. */ + cpuinfo_uarch_kryo = 0x00400102, + /** Qualcomm Falkor. */ + cpuinfo_uarch_falkor = 0x00400103, + /** Qualcomm Saphira. */ + cpuinfo_uarch_saphira = 0x00400104, + + /** Nvidia Denver. */ + cpuinfo_uarch_denver = 0x00500100, + /** Nvidia Denver 2. */ + cpuinfo_uarch_denver2 = 0x00500101, + /** Nvidia Carmel. */ + cpuinfo_uarch_carmel = 0x00500102, + + /** Samsung Exynos M1 (Exynos 8890 big cores). */ + cpuinfo_uarch_exynos_m1 = 0x00600100, + /** Samsung Exynos M2 (Exynos 8895 big cores). */ + cpuinfo_uarch_exynos_m2 = 0x00600101, + /** Samsung Exynos M3 (Exynos 9810 big cores). */ + cpuinfo_uarch_exynos_m3 = 0x00600102, + /** Samsung Exynos M4 (Exynos 9820 big cores). */ + cpuinfo_uarch_exynos_m4 = 0x00600103, + /** Samsung Exynos M5 (Exynos 9830 big cores). */ + cpuinfo_uarch_exynos_m5 = 0x00600104, + + /* Deprecated synonym for Cortex-A76 */ + cpuinfo_uarch_cortex_a76ae = 0x00300376, + /* Deprecated names for Exynos. */ + cpuinfo_uarch_mongoose_m1 = 0x00600100, + cpuinfo_uarch_mongoose_m2 = 0x00600101, + cpuinfo_uarch_meerkat_m3 = 0x00600102, + cpuinfo_uarch_meerkat_m4 = 0x00600103, + + /** Apple A6 and A6X processors. */ + cpuinfo_uarch_swift = 0x00700100, + /** Apple A7 processor. */ + cpuinfo_uarch_cyclone = 0x00700101, + /** Apple A8 and A8X processor. */ + cpuinfo_uarch_typhoon = 0x00700102, + /** Apple A9 and A9X processor. */ + cpuinfo_uarch_twister = 0x00700103, + /** Apple A10 and A10X processor. */ + cpuinfo_uarch_hurricane = 0x00700104, + /** Apple A11 processor (big cores). */ + cpuinfo_uarch_monsoon = 0x00700105, + /** Apple A11 processor (little cores). */ + cpuinfo_uarch_mistral = 0x00700106, + /** Apple A12 processor (big cores). */ + cpuinfo_uarch_vortex = 0x00700107, + /** Apple A12 processor (little cores). */ + cpuinfo_uarch_tempest = 0x00700108, + /** Apple A13 processor (big cores). */ + cpuinfo_uarch_lightning = 0x00700109, + /** Apple A13 processor (little cores). */ + cpuinfo_uarch_thunder = 0x0070010A, + /** Apple M1 processor (big cores). */ + cpuinfo_uarch_firestorm = 0x0070010B, + /** Apple M1 processor (little cores). */ + cpuinfo_uarch_icestorm = 0x0070010C, + + /** Cavium ThunderX. */ + cpuinfo_uarch_thunderx = 0x00800100, + /** Cavium ThunderX2 (originally Broadcom Vulkan). */ + cpuinfo_uarch_thunderx2 = 0x00800200, + + /** Marvell PJ4. */ + cpuinfo_uarch_pj4 = 0x00900100, + + /** Broadcom Brahma B15. */ + cpuinfo_uarch_brahma_b15 = 0x00A00100, + /** Broadcom Brahma B53. */ + cpuinfo_uarch_brahma_b53 = 0x00A00101, + + /** Applied Micro X-Gene. */ + cpuinfo_uarch_xgene = 0x00B00100, + + /* Hygon Dhyana (a modification of AMD Zen for Chinese market). */ + cpuinfo_uarch_dhyana = 0x01000100, + + /** HiSilicon TaiShan v110 (Huawei Kunpeng 920 series processors). */ + cpuinfo_uarch_taishan_v110 = 0x00C00100, +}; + +#if defined(CPUIDINFO_ARCH_ARM) + +#define CPUINFO_ARM_MIDR_IMPLEMENTER_MASK UINT32_C(0xFF000000) +#define CPUINFO_ARM_MIDR_VARIANT_MASK UINT32_C(0x00F00000) +#define CPUINFO_ARM_MIDR_ARCHITECTURE_MASK UINT32_C(0x000F0000) +#define CPUINFO_ARM_MIDR_PART_MASK UINT32_C(0x0000FFF0) +#define CPUINFO_ARM_MIDR_REVISION_MASK UINT32_C(0x0000000F) + +#define CPUINFO_ARM_MIDR_IMPLEMENTER_OFFSET 24 +#define CPUINFO_ARM_MIDR_VARIANT_OFFSET 20 +#define CPUINFO_ARM_MIDR_ARCHITECTURE_OFFSET 16 +#define CPUINFO_ARM_MIDR_PART_OFFSET 4 +#define CPUINFO_ARM_MIDR_REVISION_OFFSET 0 + +inline static uint32_t midr_get_implementer(uint32_t midr) { + return (midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) >> CPUINFO_ARM_MIDR_IMPLEMENTER_OFFSET; +} + +inline static uint32_t midr_get_part(uint32_t midr) { + return (midr & CPUINFO_ARM_MIDR_PART_MASK) >> CPUINFO_ARM_MIDR_PART_OFFSET; +} + +inline static uint32_t midr_get_variant(uint32_t midr) { + return (midr & CPUINFO_ARM_MIDR_VARIANT_MASK) >> CPUINFO_ARM_MIDR_VARIANT_OFFSET; +} + +static void decodeMIDR( + uint32_t midr, + uint32_t uarch[1]) { + switch (midr_get_implementer(midr)) { + case 'A': + switch (midr_get_part(midr)) { + //#if defined(_M_ARM) || defined(__arm__) + case 0xC05: + *uarch = cpuinfo_uarch_cortex_a5; + break; + case 0xC07: + *uarch = cpuinfo_uarch_cortex_a7; + break; + case 0xC08: + *uarch = cpuinfo_uarch_cortex_a8; + break; + case 0xC09: + *uarch = cpuinfo_uarch_cortex_a9; + break; + case 0xC0C: + *uarch = cpuinfo_uarch_cortex_a12; + break; + case 0xC0E: + *uarch = cpuinfo_uarch_cortex_a17; + break; + case 0xC0D: + /* + * Rockchip RK3288 only. + * Core information is ambiguous: some sources specify Cortex-A12, others - Cortex-A17. + * Assume it is Cortex-A12. + */ + *uarch = cpuinfo_uarch_cortex_a12; + break; + case 0xC0F: + *uarch = cpuinfo_uarch_cortex_a15; + break; + //#endif /* ARM */ + case 0xD01: + *uarch = cpuinfo_uarch_cortex_a32; + break; + case 0xD03: + *uarch = cpuinfo_uarch_cortex_a53; + break; + case 0xD04: + *uarch = cpuinfo_uarch_cortex_a35; + break; + case 0xD05: + // Note: use Variant, not Revision, field + *uarch = (midr & CPUINFO_ARM_MIDR_VARIANT_MASK) == 0 ? cpuinfo_uarch_cortex_a55r0 : cpuinfo_uarch_cortex_a55; + break; + case 0xD06: + *uarch = cpuinfo_uarch_cortex_a65; + break; + case 0xD07: + *uarch = cpuinfo_uarch_cortex_a57; + break; + case 0xD08: + *uarch = cpuinfo_uarch_cortex_a72; + break; + case 0xD09: + *uarch = cpuinfo_uarch_cortex_a73; + break; + case 0xD0A: + *uarch = cpuinfo_uarch_cortex_a75; + break; + case 0xD0B: + *uarch = cpuinfo_uarch_cortex_a76; + break; + //#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__) + case 0xD0C: + *uarch = cpuinfo_uarch_neoverse_n1; + break; + //#endif /* ARM64 && !defined(__ANDROID__) */ + case 0xD0D: + *uarch = cpuinfo_uarch_cortex_a77; + break; + case 0xD0E: /* Cortex-A76AE */ + *uarch = cpuinfo_uarch_cortex_a76; + break; + case 0xD41: /* Cortex-A78 */ + *uarch = cpuinfo_uarch_cortex_a78; + break; + case 0xD44: /* Cortex-X1 */ + *uarch = cpuinfo_uarch_cortex_x1; + break; + //#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__) + case 0xD4A: + *uarch = cpuinfo_uarch_neoverse_e1; + break; + //#endif /* ARM64 && !defined(__ANDROID__) */ + default: + switch (midr_get_part(midr) >> 8) { + //#if defined(_M_ARM) || defined(__arm__) + case 7: + *uarch = cpuinfo_uarch_arm7; + break; + case 9: + *uarch = cpuinfo_uarch_arm9; + break; + case 11: + *uarch = cpuinfo_uarch_arm11; + break; + //#endif /* ARM */ + default: + LOGS_DEFAULT(WARNING) << "unknown ARM CPU part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + } + break; + case 'B': + switch (midr_get_part(midr)) { + case 0x00F: + *uarch = cpuinfo_uarch_brahma_b15; + break; + case 0x100: + *uarch = cpuinfo_uarch_brahma_b53; + break; + //#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__) + case 0x516: + /* Broadcom Vulkan was sold to Cavium before it reached the market, so we identify it as Cavium ThunderX2 */ + *uarch = cpuinfo_uarch_thunderx2; + break; + //#endif + default: + LOGS_DEFAULT(WARNING) << "unknown Broadcom CPU part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + break; + //#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__) + case 'C': + switch (midr_get_part(midr)) { + case 0x0A0: /* ThunderX */ + case 0x0A1: /* ThunderX 88XX */ + case 0x0A2: /* ThunderX 81XX */ + case 0x0A3: /* ThunderX 83XX */ + *uarch = cpuinfo_uarch_thunderx; + break; + case 0x0AF: /* ThunderX2 99XX */ + *uarch = cpuinfo_uarch_thunderx2; + break; + default: + LOGS_DEFAULT(WARNING) << "unknown Cavium CPU part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + break; + //#endif + case 'H': + switch (midr_get_part(midr)) { + //#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__) + case 0xD01: /* Kunpeng 920 series */ + *uarch = cpuinfo_uarch_taishan_v110; + break; + //#endif + case 0xD40: /* Kirin 980 Big/Medium cores -> Cortex-A76 */ + *uarch = cpuinfo_uarch_cortex_a76; + break; + default: + LOGS_DEFAULT(WARNING) << "unknown Huawei CPU part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + break; + //#if defined(_M_ARM) || defined(__arm__) + case 'i': + switch (midr_get_part(midr) >> 8) { + case 2: /* PXA 210/25X/26X */ + case 4: /* PXA 27X */ + case 6: /* PXA 3XX */ + *uarch = cpuinfo_uarch_xscale; + break; + default: + LOGS_DEFAULT(WARNING) << "unknown Intel CPU part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + break; + //#endif /* ARM */ + case 'N': + switch (midr_get_part(midr)) { + case 0x000: + *uarch = cpuinfo_uarch_denver; + break; + case 0x003: + *uarch = cpuinfo_uarch_denver2; + break; + case 0x004: + *uarch = cpuinfo_uarch_carmel; + break; + default: + LOGS_DEFAULT(WARNING) << "unknown Nvidia CPU part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + break; +#if !defined(__ANDROID__) + case 'P': + switch (midr_get_part(midr)) { + case 0x000: + *uarch = cpuinfo_uarch_xgene; + break; + default: + LOGS_DEFAULT(WARNING) << "unknown Applied Micro CPU part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + break; +#endif + case 'Q': + switch (midr_get_part(midr)) { + // #if defined(_M_ARM) || defined(__arm__) + case 0x00F: + /* Mostly Scorpions, but some Cortex A5 may report this value as well */ + //if (has_vfpv4) { + // /* Unlike Scorpion, Cortex-A5 comes with VFPv4 */ + // *vendor = cpuinfo_vendor_arm; + // *uarch = cpuinfo_uarch_cortex_a5; + //} else { + *uarch = cpuinfo_uarch_scorpion; + // } + break; + case 0x02D: /* Dual-core Scorpions */ + *uarch = cpuinfo_uarch_scorpion; + break; + case 0x04D: + /* + * Dual-core Krait: + * - r1p0 -> Krait 200 + * - r1p4 -> Krait 200 + * - r2p0 -> Krait 300 + */ + case 0x06F: + /* + * Quad-core Krait: + * - r0p1 -> Krait 200 + * - r0p2 -> Krait 200 + * - r1p0 -> Krait 300 + * - r2p0 -> Krait 400 (Snapdragon 800 MSMxxxx) + * - r2p1 -> Krait 400 (Snapdragon 801 MSMxxxxPRO) + * - r3p1 -> Krait 450 + */ + *uarch = cpuinfo_uarch_krait; + break; + //#endif /* ARM */ + case 0x201: /* Qualcomm Snapdragon 821: Low-power Kryo "Silver" */ + case 0x205: /* Qualcomm Snapdragon 820 & 821: High-performance Kryo "Gold" */ + case 0x211: /* Qualcomm Snapdragon 820: Low-power Kryo "Silver" */ + *uarch = cpuinfo_uarch_kryo; + break; + case 0x800: /* High-performance Kryo 260 (r10p2) / Kryo 280 (r10p1) "Gold" -> Cortex-A73 */ + *uarch = cpuinfo_uarch_cortex_a73; + break; + case 0x801: /* Low-power Kryo 260 / 280 "Silver" -> Cortex-A53 */ + *uarch = cpuinfo_uarch_cortex_a53; + break; + case 0x802: /* High-performance Kryo 385 "Gold" -> Cortex-A75 */ + *uarch = cpuinfo_uarch_cortex_a75; + break; + case 0x803: /* Low-power Kryo 385 "Silver" -> Cortex-A55r0 */ + *uarch = cpuinfo_uarch_cortex_a55r0; + break; + case 0x804: /* High-performance Kryo 485 "Gold" / "Gold Prime" -> Cortex-A76 */ + *uarch = cpuinfo_uarch_cortex_a76; + break; + case 0x805: /* Low-performance Kryo 485 "Silver" -> Cortex-A55 */ + *uarch = cpuinfo_uarch_cortex_a55; + break; + //#if (defined(_M_ARM64) || defined(__aarch64__)) && !defined(__ANDROID__) + case 0xC00: + *uarch = cpuinfo_uarch_falkor; + break; + case 0xC01: + *uarch = cpuinfo_uarch_saphira; + break; + //#endif /* ARM64 && !defined(__ANDROID__) */ + default: + LOGS_DEFAULT(WARNING) << "unknown Qualcomm CPU part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + break; + case 'S': + switch (midr & (CPUINFO_ARM_MIDR_VARIANT_MASK | CPUINFO_ARM_MIDR_PART_MASK)) { + case 0x00100010: + /* + * Exynos 8890 MIDR = 0x531F0011, assume Exynos M1 has: + * - CPU variant 0x1 + * - CPU part 0x001 + */ + *uarch = cpuinfo_uarch_exynos_m1; + break; + case 0x00400010: + /* + * Exynos 8895 MIDR = 0x534F0010, assume Exynos M2 has: + * - CPU variant 0x4 + * - CPU part 0x001 + */ + *uarch = cpuinfo_uarch_exynos_m2; + break; + case 0x00100020: + /* + * Exynos 9810 MIDR = 0x531F0020, assume Exynos M3 has: + * - CPU variant 0x1 + * - CPU part 0x002 + */ + *uarch = cpuinfo_uarch_exynos_m3; + break; + case 0x00100030: + /* + * Exynos 9820 MIDR = 0x531F0030, assume Exynos M4 has: + * - CPU variant 0x1 + * - CPU part 0x003 + */ + *uarch = cpuinfo_uarch_exynos_m4; + break; + case 0x00100040: + /* + * Exynos 9820 MIDR = 0x531F0040, assume Exynos M5 has: + * - CPU variant 0x1 + * - CPU part 0x004 + */ + *uarch = cpuinfo_uarch_exynos_m5; + break; + default: + LOGS_DEFAULT(WARNING) << "unknown Samsung CPU variant 0x" + << std::hex << midr_get_variant(midr) << " part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + break; + //#if defined(_M_ARM) || defined(__arm__) + case 'V': + switch (midr_get_part(midr)) { + case 0x581: /* PJ4 / PJ4B */ + case 0x584: /* PJ4B-MP / PJ4C */ + *uarch = cpuinfo_uarch_pj4; + break; + default: + LOGS_DEFAULT(WARNING) << "unknown Marvell CPU part 0x" << std::hex << midr_get_part(midr) << " ignored"; + } + break; + //#endif /* ARM */ + default: + LOGS_DEFAULT(WARNING) << "unknown CPU uarch from MIDR value: 0x" << std::hex << midr; + } +} + +#endif // arm or arm64 diff --git a/onnxruntime/core/mlas/lib/convsym.cpp b/onnxruntime/core/mlas/lib/convsym.cpp index b1ca9a1d51..a667fe4048 100644 --- a/onnxruntime/core/mlas/lib/convsym.cpp +++ b/onnxruntime/core/mlas/lib/convsym.cpp @@ -482,7 +482,7 @@ MlasConvSym( // s8s8 under ARM64 #if defined(MLAS_TARGET_ARM64) const auto Kernel = - (Params.InputIsSigned && (GetMlasPlatform().GetCoreType() == mlas_core_little)) + (Params.InputIsSigned && MLAS_CPUIDINFO::GetCPUIDInfo().IsCurrentCoreArmv8NarrowLd()) ? ConvSymDispatch->KernelLittle : ConvSymDispatch->Kernel; #else diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 0a60f948fd..df00121c36 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -104,22 +104,11 @@ Abstract: #if !defined(BUILD_MLAS_NO_ONNXRUNTIME) #include "core/platform/threadpool.h" -#if defined(MLAS_TARGET_ARM64) && defined(__linux__) - #include "core/common/cpuid_info.h" using MLAS_CPUIDINFO = onnxruntime::CPUIDInfo; -#include -#include -#if !defined(__NR_getcpu) -#include -#endif - -#endif // MLAS_TARGET_ARM64 - #else // BUILD_MLAS_NO_ONNXRUNTIME -#if defined(MLAS_TARGET_ARM64) && defined(__linux__) class MLASCPUIDInfo { public: @@ -132,7 +121,15 @@ class MLASCPUIDInfo // ARM bool HasArmNeonDot() const { return has_arm_neon_dot_; } - int32_t GetCurrentUarch() { return -1; } + uint32_t GetCurrentCoreIdx() const { return 0xFFFFFFFF; } + + int32_t GetCurrentUarch() const { return -1; } + + int32_t GetCoreUarch(uint32_t coreId) const { return -1; } + + bool IsCoreArmv8NarrowLd(uint32_t coreId) const { return false; } + + bool IsCurrentCoreArmv8NarrowLd() const { return false; } private: MLASCPUIDInfo(); @@ -141,6 +138,45 @@ class MLASCPUIDInfo }; using MLAS_CPUIDINFO = MLASCPUIDInfo; +#if defined(MLAS_TARGET_ARM64) +/** + * @brief IDs for cpu microarchitectures. + * + * Copied from python cpuinfo package. Can't use the definition + * from cpuinfo directly as it causes lots of compilation issues + * in many platforms that we support. + */ +enum MlasUArch { + cpuinfo_uarch_unknown = 0, + + /** ARM Cortex-A32. */ + cpuinfo_uarch_cortex_a32 = 0x00300332, + /** ARM Cortex-A35. */ + cpuinfo_uarch_cortex_a35 = 0x00300335, + /** ARM Cortex-A53. */ + cpuinfo_uarch_cortex_a53 = 0x00300353, + /** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities compared to revision 1+). */ + cpuinfo_uarch_cortex_a55r0 = 0x00300354, + /** ARM Cortex-A55. */ + cpuinfo_uarch_cortex_a55 = 0x00300355, + /** ARM Cortex-A57. */ + cpuinfo_uarch_cortex_a57 = 0x00300357, + /** ARM Cortex-A65. */ + cpuinfo_uarch_cortex_a65 = 0x00300365, + /** ARM Cortex-A72. */ + cpuinfo_uarch_cortex_a72 = 0x00300372, + /** ARM Cortex-A73. */ + cpuinfo_uarch_cortex_a73 = 0x00300373, + /** ARM Cortex-A75. */ + cpuinfo_uarch_cortex_a75 = 0x00300375, + /** ARM Cortex-A76. */ + cpuinfo_uarch_cortex_a76 = 0x00300376, + /** ARM Cortex-A77. */ + cpuinfo_uarch_cortex_a77 = 0x00300377, + /** ARM Cortex-A78. */ + cpuinfo_uarch_cortex_a78 = 0x00300378, +}; + #endif // MLAS_TARGET_ARM64 #endif // BUILD_MLAS_NO_ONNXRUNTIME @@ -785,44 +821,6 @@ struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { // Environment information class. // -/** - * @brief IDs for cpu microarchitectures. - * - * Copied from python cpuinfo package. Can't use the definition - * from cpuinfo directly as it causes lots of compilation issues - * in many platforms that we support. - */ -enum MlasUArch { - cpuinfo_uarch_unknown = 0, - - /** ARM Cortex-A32. */ - cpuinfo_uarch_cortex_a32 = 0x00300332, - /** ARM Cortex-A35. */ - cpuinfo_uarch_cortex_a35 = 0x00300335, - /** ARM Cortex-A53. */ - cpuinfo_uarch_cortex_a53 = 0x00300353, - /** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities compared to revision 1+). */ - cpuinfo_uarch_cortex_a55r0 = 0x00300354, - /** ARM Cortex-A55. */ - cpuinfo_uarch_cortex_a55 = 0x00300355, - /** ARM Cortex-A57. */ - cpuinfo_uarch_cortex_a57 = 0x00300357, - /** ARM Cortex-A65. */ - cpuinfo_uarch_cortex_a65 = 0x00300365, - /** ARM Cortex-A72. */ - cpuinfo_uarch_cortex_a72 = 0x00300372, - /** ARM Cortex-A73. */ - cpuinfo_uarch_cortex_a73 = 0x00300373, - /** ARM Cortex-A75. */ - cpuinfo_uarch_cortex_a75 = 0x00300375, - /** ARM Cortex-A76. */ - cpuinfo_uarch_cortex_a76 = 0x00300376, - /** ARM Cortex-A77. */ - cpuinfo_uarch_cortex_a77 = 0x00300377, - /** ARM Cortex-A78. */ - cpuinfo_uarch_cortex_a78 = 0x00300378, -}; - enum MlasCoreType { mlas_core_unknown = 0, mlas_core_little = 2, mlas_core_big = 3 }; @@ -891,51 +889,6 @@ struct MLAS_PLATFORM { static constexpr int32_t MaximumThreadCount = MLAS_MAXIMUM_THREAD_COUNT; #endif -#if defined(MLAS_TARGET_ARM64) && defined(__linux__) - // TODO!! implement uarch detection in Windows - std::vector mlas_coretype_tbl; -#endif - - /** - * @return 2 current core is little core with narrow memory load (e.g. ARMv8 a53) - * 3 current core is big core with wider load (e.g. ARMv8 a72) - */ - MlasCoreType GetCoreType() - { -#if defined(MLAS_TARGET_ARM64) && defined(__linux__) - - if (mlas_coretype_tbl.size() == 0) { - // functionality missing, return default - return mlas_core_big; - } - - unsigned cpu = 0; - if (syscall(__NR_getcpu, &cpu, NULL, NULL) != 0) { - // failed to detect current core id. give up - return mlas_core_big; - } - - if (cpu >= mlas_coretype_tbl.size()) { - mlas_coretype_tbl.resize(cpu + 1, mlas_core_unknown); - } - - auto core_type = mlas_coretype_tbl[cpu]; - if (core_type == mlas_core_unknown) { - auto uarch = MLAS_CPUIDINFO::GetCPUIDInfo().GetCurrentUarch(); - if (uarch == cpuinfo_uarch_cortex_a53 || uarch == cpuinfo_uarch_cortex_a55r0 || - uarch == cpuinfo_uarch_cortex_a55) { - core_type = mlas_core_little; - } else { - core_type = mlas_core_big; - } - mlas_coretype_tbl[cpu] = core_type; - } - return core_type; - -#else - return mlas_core_big; -#endif - } }; inline diff --git a/onnxruntime/core/mlas/lib/platform.cpp b/onnxruntime/core/mlas/lib/platform.cpp index 3dbff2cfa8..ab570573d6 100644 --- a/onnxruntime/core/mlas/lib/platform.cpp +++ b/onnxruntime/core/mlas/lib/platform.cpp @@ -26,11 +26,21 @@ Abstract: #if defined(MLAS_TARGET_ARM64) #if defined(_WIN32) + // N.B. Support building with downlevel versions of the Windows SDK. #ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE #define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43 #endif + +#if defined(BUILD_MLAS_NO_ONNXRUNTIME) +MLASCPUIDInfo::MLASCPUIDInfo() +{ + has_arm_neon_dot_ = (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0); +} +#endif + #elif defined(__linux__) + #include #include // N.B. Support building with older versions of asm/hwcap.h that do not define @@ -43,7 +53,19 @@ Abstract: MLASCPUIDInfo::MLASCPUIDInfo() { has_arm_neon_dot_ = ((getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0); } #endif +#else + +#if defined(BUILD_MLAS_NO_ONNXRUNTIME) +MLASCPUIDInfo::MLASCPUIDInfo() {} #endif + +#endif // Windows vs Linux vs Unknown +#else // not MLAS_TARGET_ARM64 + +#if defined(BUILD_MLAS_NO_ONNXRUNTIME) +MLASCPUIDInfo::MLASCPUIDInfo() {} +#endif + #endif // MLAS_TARGET_ARM64 #ifdef MLAS_TARGET_AMD64_IX86 @@ -411,14 +433,6 @@ Return Value: #endif // __linux__ #endif // MLAS_TARGET_POWER - // Init the table describing the type (big or litte) of each core -#if defined(MLAS_TARGET_ARM64) && defined(__linux__) - // TODO!! implemente core uarch detection in Windows - auto tbl_size = std::thread::hardware_concurrency(); - if (tbl_size > 0) { - mlas_coretype_tbl.resize(tbl_size, mlas_core_unknown); - } -#endif } size_t diff --git a/onnxruntime/core/mlas/lib/qgemm.cpp b/onnxruntime/core/mlas/lib/qgemm.cpp index 108a36534a..8cc9d695f9 100644 --- a/onnxruntime/core/mlas/lib/qgemm.cpp +++ b/onnxruntime/core/mlas/lib/qgemm.cpp @@ -209,9 +209,9 @@ MlasSymmQgemmBatch( if (ThreadPool == nullptr) { // So our caller handles threaded job partition. // Call single threaded operation directly - auto uarch = GetMlasPlatform().GetCoreType(); + auto uarch = MLAS_CPUIDINFO::GetCPUIDInfo().IsCurrentCoreArmv8NarrowLd(); MLAS_SYMM_QGEMM_OPERATION* operation = - uarch == mlas_core_little ? dispatch->LitOperation : dispatch->BigOperation; + uarch ? dispatch->LitOperation : dispatch->BigOperation; for (size_t gemm_i = 0; gemm_i < BatchN; gemm_i++) { auto Data = &DataParams[gemm_i]; @@ -260,9 +260,9 @@ MlasSymmQgemmBatch( ThreadsPerGemm = ThreadCountM * ThreadCountN; MlasTrySimpleParallel(ThreadPool, ThreadsPerGemm * BatchN, [&](ptrdiff_t tid) { - auto uarch = GetMlasPlatform().GetCoreType(); + auto uarch = MLAS_CPUIDINFO::GetCPUIDInfo().IsCurrentCoreArmv8NarrowLd(); MLAS_SYMM_QGEMM_OPERATION* operation = - uarch == mlas_core_little ? dispatch->LitOperation : dispatch->BigOperation; + uarch ? dispatch->LitOperation : dispatch->BigOperation; const auto gemm_i = tid / ThreadsPerGemm; const auto blk_i = tid % ThreadsPerGemm;