From 14eae293bf84956f371bfac50628e15b73751940 Mon Sep 17 00:00:00 2001 From: Tracy Sharpe <42477615+tracysh@users.noreply.github.com> Date: Wed, 28 Aug 2019 11:27:16 -0700 Subject: [PATCH] remove @PCGOTREL x64 usage (#1707) Avoid the need for @PCGOTREL relocations by annotating MLAS global data shared with assembly modules with attribute(visibility("hidden")). --- .../core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm | 2 ++ onnxruntime/core/mlas/lib/erf.cpp | 2 +- onnxruntime/core/mlas/lib/logistic.cpp | 2 +- onnxruntime/core/mlas/lib/mlasi.h | 16 +++++++++++++--- onnxruntime/core/mlas/lib/sgemm.cpp | 2 +- onnxruntime/core/mlas/lib/tanh.cpp | 2 +- onnxruntime/core/mlas/lib/x86_64/ErfKernelFma3.S | 8 ++++---- .../core/mlas/lib/x86_64/LogisticKernelFma3.S | 5 ++--- .../core/mlas/lib/x86_64/QgemmU8U8KernelAvx2.S | 10 +++++----- .../mlas/lib/x86_64/QgemmU8U8KernelAvx512BW.S | 2 +- .../lib/x86_64/QgemmU8U8KernelAvx512Common.h | 2 +- .../mlas/lib/x86_64/QgemmU8U8KernelAvx512Vnni.S | 2 +- .../core/mlas/lib/x86_64/SgemmKernelAvx.S | 15 ++++++--------- .../core/mlas/lib/x86_64/SgemmKernelFma3.S | 9 +++------ .../core/mlas/lib/x86_64/SgemmKernelM1Avx.S | 5 ++--- .../mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S | 5 ++--- .../core/mlas/lib/x86_64/TanhKernelFma3.S | 5 ++--- 17 files changed, 48 insertions(+), 46 deletions(-) diff --git a/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm b/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm index f1175b327d..365348a14d 100644 --- a/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm +++ b/onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm @@ -13,6 +13,8 @@ ; This module implements the kernels for the quantized integer matrix/matrix ; multiply operation (QGEMM). ; +; This implementation uses AVX2 instructions. +; ;-- .xlist diff --git a/onnxruntime/core/mlas/lib/erf.cpp b/onnxruntime/core/mlas/lib/erf.cpp index 12fd0a368d..c1f4a7e6c2 100644 --- a/onnxruntime/core/mlas/lib/erf.cpp +++ b/onnxruntime/core/mlas/lib/erf.cpp @@ -29,7 +29,7 @@ Abstract: // Bundles the constants for use by kernels written in assembly. // -extern "C" const struct { +MLAS_INTERNAL_DATA const struct { float ErfUpperAbsRange; float ErfSplitBoundary; float ErfSMALL_P0; diff --git a/onnxruntime/core/mlas/lib/logistic.cpp b/onnxruntime/core/mlas/lib/logistic.cpp index 03061bb6ba..9e657f1892 100644 --- a/onnxruntime/core/mlas/lib/logistic.cpp +++ b/onnxruntime/core/mlas/lib/logistic.cpp @@ -26,7 +26,7 @@ Abstract: // Bundles the floating point constants for use by kernels written in assembly. // -extern "C" const struct { +MLAS_INTERNAL_DATA const struct { float LowerRange; float UpperRange; float alpha_9; diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 05171f9d6b..40150e5f44 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -55,6 +55,18 @@ Abstract: #define MLAS_FORCEINLINE __attribute__ ((always_inline)) inline #endif +// +// Macro to tag globals as internal data shared with kernels written in +// assembly. These globals are marked with having hidden visibility to avoid +// needing to access the data through the global object table. +// + +#if defined(_MSC_VER) +#define MLAS_INTERNAL_DATA extern "C" +#else +#define MLAS_INTERNAL_DATA extern "C" __attribute ((visibility("hidden"))) +#endif + // // Macro to suppress unreferenced parameter warnings. // @@ -521,9 +533,7 @@ MlasGetMaximumThreadCount( } #endif -#if defined(MLAS_USE_WIN32_THREADPOOL) - return MlasPlatform.MaximumThreadCount; -#elif _OPENMP +#if defined(_OPENMP) return (omp_get_num_threads() == 1) ? omp_get_max_threads() : 1; #else return 1; diff --git a/onnxruntime/core/mlas/lib/sgemm.cpp b/onnxruntime/core/mlas/lib/sgemm.cpp index f436f17e6c..5250c84487 100644 --- a/onnxruntime/core/mlas/lib/sgemm.cpp +++ b/onnxruntime/core/mlas/lib/sgemm.cpp @@ -55,7 +55,7 @@ struct MLAS_SGEMM_WORK_BLOCK { // Stores a vector to build a conditional load/store mask for vmaskmovps. // -extern "C" MLAS_DECLSPEC_ALIGN(const uint32_t MlasMaskMoveAvx[8], 8 * sizeof(float)) = { 0, 1, 2, 3, 4, 5, 6, 7 }; +MLAS_INTERNAL_DATA MLAS_DECLSPEC_ALIGN(const uint32_t MlasMaskMoveAvx[8], 8 * sizeof(float)) = { 0, 1, 2, 3, 4, 5, 6, 7 }; #endif diff --git a/onnxruntime/core/mlas/lib/tanh.cpp b/onnxruntime/core/mlas/lib/tanh.cpp index 430afdf60e..2fbeaef3d9 100644 --- a/onnxruntime/core/mlas/lib/tanh.cpp +++ b/onnxruntime/core/mlas/lib/tanh.cpp @@ -26,7 +26,7 @@ Abstract: // Bundles the floating point constants for use by kernels written in assembly. // -extern "C" const struct { +MLAS_INTERNAL_DATA const struct { float LowerRange; float UpperRange; float alpha_13; diff --git a/onnxruntime/core/mlas/lib/x86_64/ErfKernelFma3.S b/onnxruntime/core/mlas/lib/x86_64/ErfKernelFma3.S index 29518fb911..92b7976d7d 100644 --- a/onnxruntime/core/mlas/lib/x86_64/ErfKernelFma3.S +++ b/onnxruntime/core/mlas/lib/x86_64/ErfKernelFma3.S @@ -26,6 +26,7 @@ Abstract: // // Structure layout for the erf constants block. // + .equ ErfUpperAbsRange, 0 .equ ErfSplitBoundary, 4 .equ ErfSMALL_P0, 8 @@ -68,7 +69,7 @@ Abstract: .equ ErfBuffer1, 128 .equ ErfKernelFrame_CountN, 256 .equ ErfKernelFrame_ReturnAddress, 256+8 - + /*++ Routine Description: @@ -92,7 +93,7 @@ Return Value: .globl C_UNDERSCORE(MlasErfKernelFma3) C_UNDERSCORE(MlasErfKernelFma3): sub rsp,ErfKernelFrame_ReturnAddress - mov rax,C_UNDERSCORE(MlasErfConstants)@GOTPCREL[rip] + lea rax,C_UNDERSCORE(MlasErfConstants)[rip] sub rdx,8*4 jb .LErfProcessRemainingCount @@ -376,10 +377,9 @@ C_UNDERSCORE(MlasErfKernelFma3): .LErfProcess1x8: mov DWORD PTR ErfKernelFrame_CountN[rsp],edx - mov rcx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm3,DWORD PTR ErfKernelFrame_CountN[rsp] - vpcmpgtd ymm3,ymm3,YMMWORD PTR [rcx] + vpcmpgtd ymm3,ymm3,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] vbroadcastss ymm15,ErfNegZero[rax] vmaskmovps ymm0,ymm3,YMMWORD PTR [rdi] # original input vx0 diff --git a/onnxruntime/core/mlas/lib/x86_64/LogisticKernelFma3.S b/onnxruntime/core/mlas/lib/x86_64/LogisticKernelFma3.S index 8b7b27dcbb..243b355398 100644 --- a/onnxruntime/core/mlas/lib/x86_64/LogisticKernelFma3.S +++ b/onnxruntime/core/mlas/lib/x86_64/LogisticKernelFma3.S @@ -72,7 +72,7 @@ Return Value: .globl C_UNDERSCORE(MlasLogisticKernelFma3) C_UNDERSCORE(MlasLogisticKernelFma3): - mov rax,C_UNDERSCORE(MlasLogisticConstants)@GOTPCREL[rip] + lea rax,C_UNDERSCORE(MlasLogisticConstants)[rip] vbroadcastss ymm4,LogisticConstants_LowerRange[rax] vbroadcastss ymm5,LogisticConstants_UpperRange[rax] vbroadcastss ymm6,LogisticConstants_alpha_9[rax] @@ -120,9 +120,8 @@ C_UNDERSCORE(MlasLogisticKernelFma3): add rdx,8 # correct for over-subtract above jz .LExitKernel mov DWORD PTR LogisticKernelFrame_CountN[rsp],edx - mov rcx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm2,DWORD PTR LogisticKernelFrame_CountN[rsp] - vpcmpgtd ymm2,ymm2,YMMWORD PTR [rcx] + vpcmpgtd ymm2,ymm2,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] vmaskmovps ymm0,ymm2,YMMWORD PTR [rdi] vmaxps ymm0,ymm4,ymm0 # clamp lower bound vminps ymm0,ymm5,ymm0 # clamp upper bound diff --git a/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx2.S b/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx2.S index 94aed7b7e1..8837be62c5 100644 --- a/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx2.S +++ b/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx2.S @@ -6,13 +6,15 @@ Licensed under the MIT License. Module Name: - QgemmU8U8KernelAvx2.S + QgemmU8U8KernelAvx2.s Abstract: This module implements the kernels for the quantized integer matrix/matrix multiply operation (QGEMM). + This implementation uses AVX2 instructions. + --*/ #include "asmmacro.h" @@ -122,9 +124,8 @@ C_UNDERSCORE(MlasGemmU8U8CopyPackAAvx2): inc eax shr eax,1 # align unaligned count to pair count mov DWORD PTR .LGemmU8U8CopyPackAFrame_mask[rsp],eax - mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vpbroadcastd ymm9,DWORD PTR .LGemmU8U8CopyPackAFrame_mask[rsp] - vpcmpgtd ymm9,ymm9,YMMWORD PTR [rbp] + vpcmpgtd ymm9,ymm9,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] // // Zero initialize the padded stack buffers. @@ -973,9 +974,8 @@ Implicit Arguments: .LOutputMasked8xNBlock.\RowCount\(): mov DWORD PTR .LGemmU8U8KernelFrame_mask[rsp],r9d - mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vpbroadcastd ymm0,DWORD PTR .LGemmU8U8KernelFrame_mask[rsp] - vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp] + vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] test r14b,r14b # ZeroMode? jnz .LSkipAccumulateOutputMasked8xNBlock.\RowCount\() EmitIfCountGE \RowCount\(), 1, "vpmaskmovd ymm4,ymm0,YMMWORD PTR [rdx]" diff --git a/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512BW.S b/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512BW.S index 727760ec98..bacb29a9a1 100644 --- a/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512BW.S +++ b/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512BW.S @@ -6,7 +6,7 @@ Licensed under the MIT License. Module Name: - QgemmU8U8KernelAvx512BW.S + QgemmU8U8KernelAvx512BW.s Abstract: diff --git a/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Common.h b/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Common.h index a2aa1839f7..3abd87b7ce 100644 --- a/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Common.h +++ b/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Common.h @@ -6,7 +6,7 @@ Licensed under the MIT License. Module Name: - QgemmU8U8KernelAvx512Common.inc + QgemmU8U8KernelAvx512Common.h Abstract: diff --git a/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Vnni.S b/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Vnni.S index 8a8ff44aac..76a85427d5 100644 --- a/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Vnni.S +++ b/onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Vnni.S @@ -6,7 +6,7 @@ Licensed under the MIT License. Module Name: - QgemmU8U8KernelAvx512Vnni.asm + QgemmU8U8KernelAvx512Vnni.s Abstract: diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S index 0147f08edc..63c6d5d2c8 100644 --- a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S @@ -374,10 +374,9 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx): .L\Mode\().OutputMasked8x4Block: vmovd xmm0,r9d - mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm0,xmm0,xmm0,0 - vpcmpgtd xmm1,xmm0,XMMWORD PTR [rbp+16] - vpcmpgtd xmm0,xmm0,XMMWORD PTR [rbp] + vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] vinsertf128 ymm0,ymm0,xmm1,1 .ifeqs "\Mode\()","Add" vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx] @@ -473,10 +472,9 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx): .L\Mode\().OutputMasked8x2Block: vmovd xmm0,r9d - mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm0,xmm0,xmm0,0 - vpcmpgtd xmm1,xmm0,XMMWORD PTR [rbp+16] - vpcmpgtd xmm0,xmm0,XMMWORD PTR [rbp] + vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] vinsertf128 ymm0,ymm0,xmm1,1 .ifeqs "\Mode\()","Add" vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx] @@ -540,10 +538,9 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx): .L\Mode\().OutputMasked8x1Block: vmovd xmm0,r9d - mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm0,xmm0,xmm0,0 - vpcmpgtd xmm1,xmm0,XMMWORD PTR [rbp+16] - vpcmpgtd xmm0,xmm0,XMMWORD PTR [rbp] + vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] vinsertf128 ymm0,ymm0,xmm1,1 .ifeqs "\Mode\()","Add" vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx] diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S index cfeceb6be3..a7382f8979 100644 --- a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S @@ -435,9 +435,8 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3): .L\Mode\().OutputMasked8x6Block: mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d - mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask] - vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp] + vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] .ifeqs "\Mode\()","Add" vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx] vmaskmovps ymm6,ymm0,YMMWORD PTR [rdx+rax] @@ -550,9 +549,8 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3): .L\Mode\().OutputMasked8x3Block: mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d - mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask] - vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp] + vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] .ifeqs "\Mode\()","Add" vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx] vmaskmovps ymm6,ymm0,YMMWORD PTR [rdx+rax] @@ -653,9 +651,8 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3): .L\Mode\().OutputMasked8x1Block: mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d - mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask] - vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp] + vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] .ifeqs "\Mode\()","Add" vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx] vfmadd213ps ymm5,ymm2,ymm4 diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S index 28fca0e956..86bc82b230 100644 --- a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S @@ -80,10 +80,9 @@ C_UNDERSCORE(MlasSgemmKernelM1Avx): mov eax,r8d and eax,7 vmovd xmm7,eax - mov rbx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm7,xmm7,xmm7,0 - vpcmpgtd xmm6,xmm7,XMMWORD PTR [rbx+16] - vpcmpgtd xmm7,xmm7,XMMWORD PTR [rbx] + vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] vinsertf128 ymm7,ymm7,xmm6,1 // diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S index 8d5ff17f90..86bc9209fa 100644 --- a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S @@ -79,10 +79,9 @@ C_UNDERSCORE(MlasSgemmKernelM1TransposeBAvx): mov eax,ecx and eax,7 vmovd xmm7,eax - mov rbx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm7,xmm7,xmm7,0 - vpcmpgtd xmm6,xmm7,XMMWORD PTR [rbx+16] - vpcmpgtd xmm7,xmm7,XMMWORD PTR [rbx] + vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] + vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] vinsertf128 ymm7,ymm7,xmm6,1 // diff --git a/onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S b/onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S index 61bbef5c91..dd5584648d 100644 --- a/onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S +++ b/onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S @@ -72,7 +72,7 @@ Return Value: .globl C_UNDERSCORE(MlasTanhKernelFma3) C_UNDERSCORE(MlasTanhKernelFma3): - mov rax,C_UNDERSCORE(MlasTanhConstants)@GOTPCREL[rip] + lea rax,C_UNDERSCORE(MlasTanhConstants)[rip] vbroadcastss ymm4,TanhConstants_LowerRange[rax] vbroadcastss ymm5,TanhConstants_UpperRange[rax] vbroadcastss ymm6,TanhConstants_alpha_13[rax] @@ -116,9 +116,8 @@ C_UNDERSCORE(MlasTanhKernelFma3): add rdx,8 # correct for over-subtract above jz .LExitKernel mov DWORD PTR TanhKernelFrame_CountN[rsp],edx - mov rcx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm2,DWORD PTR TanhKernelFrame_CountN[rsp] - vpcmpgtd ymm2,ymm2,YMMWORD PTR [rcx] + vpcmpgtd ymm2,ymm2,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] vmaskmovps ymm0,ymm2,YMMWORD PTR [rdi] vmaxps ymm0,ymm4,ymm0 # clamp lower bound vminps ymm0,ymm5,ymm0 # clamp upper bound